2024-09-14T03:25:49.909 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T03:25:49.936 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:25:50.036 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7904532' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 8038 sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded targets: smithi001.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHv9ETz7IPs2patoth8GvJ+9cL0nWAF6hGkJlPwdaGUEPcm6ib2JbuA8JMRVu4xrmezuwB13EvEsAssFfj2vyRw= smithi012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCcEbXVIB8VnKeIu+2yDSDtUr2UA8z+9aVsqn/REAq/9cFsmEiVWRKfRVY7WTtgQqoejfS5fTZ3PpFfYOdHsTtw= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-14_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-14T03:25:50.036 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa; will attempt to use it 2024-09-14T03:25:50.037 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks 2024-09-14T03:25:50.037 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T03:25:50.038 INFO:teuthology.task.internal:Checking packages... 2024-09-14T03:25:50.060 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8' 2024-09-14T03:25:50.061 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:25:50.061 INFO:teuthology.packaging:ref: None 2024-09-14T03:25:50.061 INFO:teuthology.packaging:tag: None 2024-09-14T03:25:50.061 INFO:teuthology.packaging:branch: main 2024-09-14T03:25:50.061 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:25:50.061 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T03:25:50.314 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T03:25:50.316 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T03:25:50.322 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T03:25:50.322 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T03:25:50.327 INFO:teuthology.task.internal:Saving configuration 2024-09-14T03:25:50.350 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T03:25:50.356 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T03:25:50.383 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi001.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:21:05.257061', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHv9ETz7IPs2patoth8GvJ+9cL0nWAF6hGkJlPwdaGUEPcm6ib2JbuA8JMRVu4xrmezuwB13EvEsAssFfj2vyRw='} 2024-09-14T03:25:50.407 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:21:05.254900', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCcEbXVIB8VnKeIu+2yDSDtUr2UA8z+9aVsqn/REAq/9cFsmEiVWRKfRVY7WTtgQqoejfS5fTZ3PpFfYOdHsTtw='} 2024-09-14T03:25:50.408 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T03:25:50.413 INFO:teuthology.task.internal:roles: ubuntu@smithi001.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-14T03:25:50.414 INFO:teuthology.task.internal:roles: ubuntu@smithi012.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-14T03:25:50.414 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T03:25:50.469 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f079da6fac0>, signals=[15]) 2024-09-14T03:25:50.469 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T03:25:50.475 INFO:teuthology.task.internal:Opening connections... 2024-09-14T03:25:50.475 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi001.front.sepia.ceph.com 2024-09-14T03:25:50.476 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi001.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:25:50.550 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi012.front.sepia.ceph.com 2024-09-14T03:25:50.551 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:25:50.628 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T03:25:50.635 DEBUG:teuthology.orchestra.run.smithi001:> uname -m 2024-09-14T03:25:50.653 INFO:teuthology.orchestra.run.smithi001.stdout:x86_64 2024-09-14T03:25:50.654 DEBUG:teuthology.orchestra.run.smithi001:> cat /etc/os-release 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:NAME="CentOS Stream" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:VERSION="9" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:ID="centos" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:VERSION_ID="9" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:25:50.712 INFO:teuthology.orchestra.run.smithi001.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:ANSI_COLOR="0;31" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:25:50.713 INFO:teuthology.orchestra.run.smithi001.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:25:50.714 INFO:teuthology.lock.ops:Updating smithi001.front.sepia.ceph.com on lock server 2024-09-14T03:25:50.733 DEBUG:teuthology.orchestra.run.smithi012:> uname -m 2024-09-14T03:25:50.752 INFO:teuthology.orchestra.run.smithi012.stdout:x86_64 2024-09-14T03:25:50.753 DEBUG:teuthology.orchestra.run.smithi012:> cat /etc/os-release 2024-09-14T03:25:50.810 INFO:teuthology.orchestra.run.smithi012.stdout:NAME="CentOS Stream" 2024-09-14T03:25:50.810 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION="9" 2024-09-14T03:25:50.810 INFO:teuthology.orchestra.run.smithi012.stdout:ID="centos" 2024-09-14T03:25:50.810 INFO:teuthology.orchestra.run.smithi012.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION_ID="9" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:ANSI_COLOR="0;31" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:25:50.811 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:25:50.812 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:25:50.812 INFO:teuthology.lock.ops:Updating smithi012.front.sepia.ceph.com on lock server 2024-09-14T03:25:50.834 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T03:25:50.843 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T03:25:50.873 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T03:25:50.873 DEBUG:teuthology.orchestra.run.smithi001:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:25:50.875 DEBUG:teuthology.orchestra.run.smithi012:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:25:50.892 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T03:25:50.899 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T03:25:50.899 DEBUG:teuthology.orchestra.run.smithi001:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:25:50.933 DEBUG:teuthology.orchestra.run.smithi012:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:25:50.963 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T03:25:51.025 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T03:25:51.039 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T03:25:51.040 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T03:25:51.040 DEBUG:teuthology.orchestra.run.smithi001:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:25:51.040 DEBUG:teuthology.orchestra.run.smithi012:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:25:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:25:51.057 DEBUG:teuthology.orchestra.run.smithi001:> uname -r 2024-09-14T03:25:51.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:25:51.058 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-09-14T03:25:51.115 INFO:teuthology.orchestra.run.smithi001.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:25:51.116 INFO:teuthology.task.kernel:Running kernel on smithi001: 5.14.0-503.el9.x86_64 2024-09-14T03:25:51.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum install -y kernel 2024-09-14T03:25:51.117 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:25:51.117 INFO:teuthology.task.kernel:Running kernel on smithi012: 5.14.0-503.el9.x86_64 2024-09-14T03:25:51.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum install -y kernel 2024-09-14T03:25:53.599 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-09-14T03:25:54.849 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 4.1 MB/s | 8.2 MB 00:02 2024-09-14T03:25:55.516 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - AppStream 27 MB/s | 20 MB 00:00 2024-09-14T03:25:58.183 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 9.3 MB/s | 20 MB 00:02 2024-09-14T03:25:59.825 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - CRB 16 MB/s | 6.5 MB 00:00 2024-09-14T03:26:01.501 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - Extras packages 45 kB/s | 19 kB 00:00 2024-09-14T03:26:02.330 INFO:teuthology.orchestra.run.smithi001.stdout:Extra Packages for Enterprise Linux 32 MB/s | 23 MB 00:00 2024-09-14T03:26:02.683 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.5 MB 00:00 2024-09-14T03:26:04.434 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 19 kB 00:00 2024-09-14T03:26:05.563 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 23 MB/s | 23 MB 00:00 2024-09-14T03:26:08.011 INFO:teuthology.orchestra.run.smithi001.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-14T03:26:10.109 INFO:teuthology.orchestra.run.smithi001.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:26:10.109 INFO:teuthology.orchestra.run.smithi001.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:26:10.201 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:26:10.208 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:26:10.209 INFO:teuthology.orchestra.run.smithi001.stdout:Install 4 Packages 2024-09-14T03:26:10.210 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:26:10.210 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 79 M 2024-09-14T03:26:10.210 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 126 M 2024-09-14T03:26:10.210 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:26:10.812 INFO:teuthology.orchestra.run.smithi001.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 41 kB/s | 23 kB 00:00 2024-09-14T03:26:11.260 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-09-14T03:26:11.962 INFO:teuthology.orchestra.run.smithi001.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 9.2 MB/s | 16 MB 00:01 2024-09-14T03:26:13.038 INFO:teuthology.orchestra.run.smithi001.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 12 MB/s | 35 MB 00:02 2024-09-14T03:26:13.358 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:26:13.358 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:26:13.455 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:26:13.465 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout:Install 4 Packages 2024-09-14T03:26:13.466 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:26:13.467 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 79 M 2024-09-14T03:26:13.467 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 126 M 2024-09-14T03:26:13.467 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:26:13.586 INFO:teuthology.orchestra.run.smithi012.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 233 kB/s | 23 kB 00:00 2024-09-14T03:26:13.978 INFO:teuthology.orchestra.run.smithi012.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 32 MB/s | 16 MB 00:00 2024-09-14T03:26:14.104 INFO:teuthology.orchestra.run.smithi001.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 8.7 MB/s | 29 MB 00:03 2024-09-14T03:26:14.105 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:26:14.105 INFO:teuthology.orchestra.run.smithi001.stdout:Total 20 MB/s | 79 MB 00:03 2024-09-14T03:26:14.370 INFO:teuthology.orchestra.run.smithi012.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 36 MB/s | 29 MB 00:00 2024-09-14T03:26:14.687 INFO:teuthology.orchestra.run.smithi012.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-14T03:26:14.687 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:26:14.687 INFO:teuthology.orchestra.run.smithi012.stdout:Total 65 MB/s | 79 MB 00:01 2024-09-14T03:26:14.817 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:26:15.195 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:26:15.195 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:26:15.403 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:26:15.817 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:26:15.817 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:26:16.160 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:26:16.160 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:26:16.773 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:26:16.774 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:26:18.767 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:26:19.375 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:26:19.482 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:26:19.580 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:26:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:26:20.154 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:26:22.359 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:26:22.483 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:26:22.957 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:26:23.092 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:26:27.614 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:26:27.659 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:26:28.213 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:26:28.259 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:26:32.796 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:26:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.211 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.256 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.561 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.600 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.600 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:27:12.600 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:27:12.600 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:27:12.602 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.959 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:12.960 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:27:12.960 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:27:12.960 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:27:13.425 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:27:13.426 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:27:13.783 DEBUG:teuthology.orchestra.run.smithi001:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:27:13.803 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:27:13.803 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:27:13.804 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:27:13.865 INFO:teuthology.orchestra.run.smithi001.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.865 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:27:13.866 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:27:13.866 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:27:13.866 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:27:13.866 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-14T03:27:13.866 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi001.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:27:13.866 DEBUG:teuthology.orchestra.run.smithi001:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:27:13.924 INFO:teuthology.orchestra.run.smithi001.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:13.925 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:27:13.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo rpm -qi grub2-tools 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Name : grub2-tools 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Epoch : 1 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Version : 2.06 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Release : 82.el9 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Architecture: x86_64 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:27:13.956 INFO:teuthology.orchestra.run.smithi001.stdout:Group : Unspecified 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Size : 8274670 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:License : GPLv3+ 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Packager : builder@centos.org 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Vendor : CentOS 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Summary : Support tools for GRUB. 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:Description : 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:27:13.957 INFO:teuthology.orchestra.run.smithi001.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:27:13.958 INFO:teuthology.orchestra.run.smithi001.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:27:13.958 INFO:teuthology.orchestra.run.smithi001.stdout:hardware devices. 2024-09-14T03:27:13.958 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:27:13.958 INFO:teuthology.orchestra.run.smithi001.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:27:13.959 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:27:13.959 INFO:teuthology.task.kernel:Updating grub on smithi001 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:27:13.959 DEBUG:teuthology.orchestra.run.smithi001:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:27:14.153 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:27:14.281 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:14.282 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:27:14.282 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:27:14.282 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:27:14.282 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:27:14.282 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-14T03:27:14.282 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi012.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:27:14.282 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:27:14.340 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:27:14.340 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:27:14.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo rpm -qi grub2-tools 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Name : grub2-tools 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Epoch : 1 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Version : 2.06 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Release : 82.el9 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Architecture: x86_64 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Group : Unspecified 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Size : 8274670 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:License : GPLv3+ 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:27:14.380 INFO:teuthology.orchestra.run.smithi012.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:Packager : builder@centos.org 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:Vendor : CentOS 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:Summary : Support tools for GRUB. 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:Description : 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:hardware devices. 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:27:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:27:14.382 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:27:14.382 INFO:teuthology.task.kernel:Updating grub on smithi012 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:27:14.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:27:14.572 INFO:teuthology.orchestra.run.smithi001.stderr:Generating grub configuration file ... 2024-09-14T03:27:15.027 INFO:teuthology.orchestra.run.smithi012.stderr:Generating grub configuration file ... 2024-09-14T03:27:16.075 INFO:teuthology.orchestra.run.smithi001.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:27:16.108 INFO:teuthology.orchestra.run.smithi001.stderr:done 2024-09-14T03:27:16.110 DEBUG:teuthology.orchestra.run.smithi001:> mktemp 2024-09-14T03:27:16.125 INFO:teuthology.orchestra.run.smithi001.stdout:/tmp/tmp.qfXcL87RwO 2024-09-14T03:27:16.125 DEBUG:teuthology.orchestra.run.smithi001:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.qfXcL87RwO 2024-09-14T03:27:16.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod 0666 /tmp/tmp.qfXcL87RwO 2024-09-14T03:27:16.328 DEBUG:teuthology.orchestra.remote:smithi001:/tmp/tmp.qfXcL87RwO is 6KB 2024-09-14T03:27:16.342 INFO:teuthology.orchestra.run.smithi012.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:27:16.377 DEBUG:teuthology.orchestra.run.smithi001:> rm -fr /tmp/tmp.qfXcL87RwO 2024-09-14T03:27:16.382 INFO:teuthology.orchestra.run.smithi012.stderr:done 2024-09-14T03:27:16.384 DEBUG:teuthology.orchestra.run.smithi012:> mktemp 2024-09-14T03:27:16.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:27:16.399 INFO:teuthology.orchestra.run.smithi012.stdout:/tmp/tmp.Tj280qAU7u 2024-09-14T03:27:16.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Tj280qAU7u 2024-09-14T03:27:16.455 INFO:teuthology.orchestra.run.smithi001.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:27:16.455 INFO:teuthology.orchestra.run.smithi001.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:27:16.455 INFO:teuthology.orchestra.run.smithi001.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:27:16.455 INFO:teuthology.orchestra.run.smithi001.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:27:16.456 DEBUG:teuthology.orchestra.run.smithi001:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:27:16.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0666 /tmp/tmp.Tj280qAU7u 2024-09-14T03:27:16.603 DEBUG:teuthology.orchestra.remote:smithi012:/tmp/tmp.Tj280qAU7u is 6KB 2024-09-14T03:27:16.623 DEBUG:teuthology.orchestra.run.smithi012:> rm -fr /tmp/tmp.Tj280qAU7u 2024-09-14T03:27:16.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:27:16.706 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:27:16.707 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:27:16.707 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:27:16.707 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:27:16.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:27:16.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo shutdown -r now 2024-09-14T03:27:16.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo shutdown -r now 2024-09-14T03:27:46.769 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-14T03:27:46.770 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi001.front.sepia.ceph.com' 2024-09-14T03:27:46.771 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi001.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:27:46.869 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-14T03:27:46.869 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:27:46.870 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:28:05.437 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.12 2024-09-14T03:28:14.445 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:28:14.446 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:28:32.829 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.12 2024-09-14T03:28:44.841 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:28:44.842 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:28:46.775 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:28:47.901 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.12 2024-09-14T03:28:55.781 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi001.front.sepia.ceph.com' 2024-09-14T03:28:55.782 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi001.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:28:56.307 DEBUG:teuthology.orchestra.run.smithi001:> true 2024-09-14T03:28:56.748 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi001.front.sepia.ceph.com' 2024-09-14T03:28:56.749 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:28:56.749 DEBUG:teuthology.orchestra.run.smithi001:> uname -r 2024-09-14T03:28:56.764 INFO:teuthology.orchestra.run.smithi001.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:28:56.764 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:28:56.764 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:28:56.764 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:28:57.765 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-14T03:28:57.765 DEBUG:teuthology.orchestra.run.smithi001:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:28:57.855 INFO:teuthology.orchestra.run.smithi001.stdout:ttyS1 2024-09-14T03:28:57.877 DEBUG:teuthology.parallel:result is None 2024-09-14T03:29:02.902 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:29:02.903 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:29:03.171 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-09-14T03:29:03.508 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:29:03.508 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:29:03.509 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-09-14T03:29:03.525 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:29:03.525 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:29:03.525 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:29:03.525 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:29:04.526 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-14T03:29:04.527 DEBUG:teuthology.orchestra.run.smithi012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:29:04.608 INFO:teuthology.orchestra.run.smithi012.stdout:ttyS1 2024-09-14T03:29:04.634 DEBUG:teuthology.parallel:result is None 2024-09-14T03:29:04.634 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T03:29:04.645 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T03:29:04.646 DEBUG:teuthology.orchestra.run.smithi001:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:29:04.649 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:29:04.668 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T03:29:04.680 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T03:29:04.699 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T03:29:04.699 DEBUG:teuthology.orchestra.run.smithi001:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:29:04.713 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:29:04.756 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T03:29:04.767 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T03:29:04.768 DEBUG:teuthology.orchestra.run.smithi001:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:29:04.783 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:29:04.811 INFO:teuthology.orchestra.run.smithi001.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:29:04.823 INFO:teuthology.orchestra.run.smithi001.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:29:04.826 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:29:04.838 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:29:04.840 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T03:29:04.850 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T03:29:04.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:29:04.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:29:04.908 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T03:29:04.920 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T03:29:04.920 DEBUG:teuthology.orchestra.run.smithi001:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:29:04.943 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:29:04.966 DEBUG:teuthology.orchestra.run.smithi001:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:29:05.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:29:05.101 DEBUG:teuthology.orchestra.run.smithi001:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:29:05.188 DEBUG:teuthology.orchestra.run.smithi001:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:29:05.234 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:29:05.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:29:05.303 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:29:05.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:29:05.400 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:29:05.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:29:05.530 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:29:05.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:29:05.600 DEBUG:teuthology.orchestra.run.smithi001:> sudo service rsyslog restart 2024-09-14T03:29:05.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo service rsyslog restart 2024-09-14T03:29:05.652 INFO:teuthology.orchestra.run.smithi001.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:29:05.688 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:29:06.209 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T03:29:06.220 INFO:teuthology.task.internal:Starting timer... 2024-09-14T03:29:06.220 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T03:29:06.237 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T03:29:06.250 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-14T03:29:06.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo service auditd rotate 2024-09-14T03:29:06.301 INFO:teuthology.orchestra.run.smithi001.stdout:Rotating logs: 2024-09-14T03:29:06.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo service auditd rotate 2024-09-14T03:29:06.350 INFO:teuthology.orchestra.run.smithi012.stdout:Rotating logs: 2024-09-14T03:29:06.352 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T03:29:06.352 DEBUG:teuthology.orchestra.run.smithi001:> /usr/sbin/getenforce 2024-09-14T03:29:06.376 INFO:teuthology.orchestra.run.smithi001.stdout:Permissive 2024-09-14T03:29:06.440 DEBUG:teuthology.orchestra.run.smithi012:> /usr/sbin/getenforce 2024-09-14T03:29:06.476 INFO:teuthology.orchestra.run.smithi012.stdout:Permissive 2024-09-14T03:29:06.476 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi001.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi012.front.sepia.ceph.com': 'permissive'} 2024-09-14T03:29:06.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:29:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:29:06.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:29:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:29:06.546 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T03:29:06.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:29:06.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:29:06.623 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T03:29:06.632 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-14T03:29:06.635 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-14T03:29:06.747 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T03:29:06.760 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T03:29:06.761 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi001.front.sepia.ceph.com,smithi012.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T03:32:56.036 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi001.front.sepia.ceph.com'), Remote(name='ubuntu@smithi012.front.sepia.ceph.com')] 2024-09-14T03:32:56.038 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi001.front.sepia.ceph.com' 2024-09-14T03:32:56.039 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi001.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:56.117 DEBUG:teuthology.orchestra.run.smithi001:> true 2024-09-14T03:32:56.202 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi001.front.sepia.ceph.com' 2024-09-14T03:32:56.203 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:32:56.203 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:56.281 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-09-14T03:32:56.377 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-14T03:32:56.378 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T03:32:56.394 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T03:32:56.394 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:32:56.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:32:56.397 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:32:56.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:32:56.430 INFO:teuthology.orchestra.run.smithi001.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:32:56.448 INFO:teuthology.orchestra.run.smithi001.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:32:56.450 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:32:56.466 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:32:56.474 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: ntpd: command not found 2024-09-14T03:32:56.486 INFO:teuthology.orchestra.run.smithi001.stdout:506 Cannot talk to daemon 2024-09-14T03:32:56.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: ntpd: command not found 2024-09-14T03:32:56.502 INFO:teuthology.orchestra.run.smithi001.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:32:56.504 INFO:teuthology.orchestra.run.smithi012.stdout:506 Cannot talk to daemon 2024-09-14T03:32:56.517 INFO:teuthology.orchestra.run.smithi001.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:32:56.520 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:32:56.535 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:32:56.574 INFO:teuthology.orchestra.run.smithi001.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:=============================================================================== 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.577 INFO:teuthology.orchestra.run.smithi001.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.590 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:32:56.595 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:32:56.595 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-09-14T03:32:56.595 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.595 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.595 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.596 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:32:56.596 INFO:teuthology.run_tasks:Running task pexec... 2024-09-14T03:32:56.606 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-14T03:32:56.606 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi001.front.sepia.ceph.com 2024-09-14T03:32:56.606 DEBUG:teuthology.orchestra.run.smithi001:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:32:56.607 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi012.front.sepia.ceph.com 2024-09-14T03:32:56.607 DEBUG:teuthology.orchestra.run.smithi012:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:32:56.986 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:32:56.998 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:56.998 INFO:teuthology.orchestra.run.smithi001.stdout: Package Architecture Version Repository Size 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout:Removing: 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout:Removing unused dependencies: 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:32:56.999 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:57.000 INFO:teuthology.orchestra.run.smithi001.stdout:Remove 2 Packages 2024-09-14T03:32:57.000 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:57.000 INFO:teuthology.orchestra.run.smithi001.stdout:Freed space: 5.5 M 2024-09-14T03:32:57.000 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:32:57.003 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:32:57.003 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:32:57.004 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:32:57.016 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Removing: 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Removing unused dependencies: 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Remove 2 Packages 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Freed space: 5.5 M 2024-09-14T03:32:57.017 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:32:57.021 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:32:57.021 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:32:57.078 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:32:57.079 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:32:57.085 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:32:57.085 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:32:57.226 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:32:57.228 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:32:57.299 INFO:teuthology.orchestra.run.smithi012.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:32:57.305 INFO:teuthology.orchestra.run.smithi001.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:32:57.364 INFO:teuthology.orchestra.run.smithi012.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:32:57.371 INFO:teuthology.orchestra.run.smithi001.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:32:57.822 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:32:57.823 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:32:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:32:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:32:58.102 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:32:58.102 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:58.103 INFO:teuthology.orchestra.run.smithi001.stdout:Removed: 2024-09-14T03:32:58.103 INFO:teuthology.orchestra.run.smithi001.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:32:58.103 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:58.103 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:32:58.205 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:32:58.205 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:58.205 INFO:teuthology.orchestra.run.smithi012.stdout:Removed: 2024-09-14T03:32:58.205 INFO:teuthology.orchestra.run.smithi012.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:32:58.205 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:58.206 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:32:58.939 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:46 ago on Sat 14 Sep 2024 03:32:12 AM UTC. 2024-09-14T03:32:59.088 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:32:59.089 INFO:teuthology.orchestra.run.smithi001.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout:Install 7 Packages 2024-09-14T03:32:59.090 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:32:59.093 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:46 ago on Sat 14 Sep 2024 03:32:12 AM UTC. 2024-09-14T03:32:59.134 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 5.2 M 2024-09-14T03:32:59.134 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 19 M 2024-09-14T03:32:59.135 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:32:59.232 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:32:59.233 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:59.233 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:32:59.234 INFO:teuthology.orchestra.run.smithi012.stdout:Install 7 Packages 2024-09-14T03:32:59.235 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:32:59.244 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 5.2 M 2024-09-14T03:32:59.244 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 19 M 2024-09-14T03:32:59.244 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:32:59.635 INFO:teuthology.orchestra.run.smithi001.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 455 kB/s | 44 kB 00:00 2024-09-14T03:32:59.660 INFO:teuthology.orchestra.run.smithi001.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 862 kB/s | 106 kB 00:00 2024-09-14T03:32:59.692 INFO:teuthology.orchestra.run.smithi012.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 466 kB/s | 44 kB 00:00 2024-09-14T03:32:59.693 INFO:teuthology.orchestra.run.smithi001.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-14T03:32:59.719 INFO:teuthology.orchestra.run.smithi001.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-14T03:32:59.726 INFO:teuthology.orchestra.run.smithi012.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 820 kB/s | 106 kB 00:00 2024-09-14T03:32:59.751 INFO:teuthology.orchestra.run.smithi012.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-14T03:32:59.761 INFO:teuthology.orchestra.run.smithi001.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.0 MB/s | 906 kB 00:00 2024-09-14T03:32:59.777 INFO:teuthology.orchestra.run.smithi012.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-09-14T03:32:59.803 INFO:teuthology.orchestra.run.smithi001.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.5 MB/s | 837 kB 00:00 2024-09-14T03:32:59.819 INFO:teuthology.orchestra.run.smithi012.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.0 MB/s | 906 kB 00:00 2024-09-14T03:32:59.861 INFO:teuthology.orchestra.run.smithi012.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.5 MB/s | 837 kB 00:00 2024-09-14T03:33:00.062 INFO:teuthology.orchestra.run.smithi001.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 9.3 MB/s | 3.2 MB 00:00 2024-09-14T03:33:00.062 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:00.062 INFO:teuthology.orchestra.run.smithi001.stdout:Total 5.6 MB/s | 5.2 MB 00:00 2024-09-14T03:33:00.136 INFO:teuthology.orchestra.run.smithi012.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 8.8 MB/s | 3.2 MB 00:00 2024-09-14T03:33:00.136 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:00.136 INFO:teuthology.orchestra.run.smithi012.stdout:Total 5.8 MB/s | 5.2 MB 00:00 2024-09-14T03:33:00.175 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:33:00.191 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:33:00.192 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:33:00.253 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:33:00.269 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:33:00.269 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:33:00.506 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:33:00.506 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:33:00.572 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:33:00.572 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:33:01.001 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:33:01.059 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:33:01.073 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:33:01.107 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:33:01.135 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:33:01.156 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:33:01.197 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:33:01.227 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:33:01.358 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:33:01.391 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:33:01.456 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:33:01.491 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:33:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:33:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:33:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:33:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:33:02.024 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:33:02.092 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:02.124 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:33:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:33:03.088 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:33:03.198 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:33:03.199 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:33:03.446 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:03.446 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:03.446 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:33:03.446 INFO:teuthology.orchestra.run.smithi001.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:33:03.447 INFO:teuthology.orchestra.run.smithi001.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:33:03.447 INFO:teuthology.orchestra.run.smithi001.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:33:03.447 INFO:teuthology.orchestra.run.smithi001.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:33:03.447 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:03.447 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:03.566 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:33:03.809 DEBUG:teuthology.parallel:result is None 2024-09-14T03:33:03.926 DEBUG:teuthology.parallel:result is None 2024-09-14T03:33:03.927 INFO:teuthology.run_tasks:Running task install... 2024-09-14T03:33:03.935 DEBUG:teuthology.task.install:project ceph 2024-09-14T03:33:03.935 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}} 2024-09-14T03:33:03.935 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:33:03.935 INFO:teuthology.task.install:Using flavor: default 2024-09-14T03:33:03.941 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:33:03.941 INFO:teuthology.task.install:extra packages: [] 2024-09-14T03:33:03.941 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:33:03.942 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:33:03.942 INFO:teuthology.packaging:ref: None 2024-09-14T03:33:03.942 INFO:teuthology.packaging:tag: None 2024-09-14T03:33:03.942 INFO:teuthology.packaging:branch: reef 2024-09-14T03:33:03.942 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:33:03.942 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:33:03.944 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:33:03.944 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:33:03.945 INFO:teuthology.packaging:ref: None 2024-09-14T03:33:03.945 INFO:teuthology.packaging:tag: None 2024-09-14T03:33:03.945 INFO:teuthology.packaging:branch: reef 2024-09-14T03:33:03.945 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:33:03.945 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:33:04.187 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/ 2024-09-14T03:33:04.187 INFO:teuthology.task.install.rpm:Package version is 18.2.4-845.ge4e227b3 2024-09-14T03:33:04.242 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/ 2024-09-14T03:33:04.242 INFO:teuthology.task.install.rpm:Package version is 18.2.4-845.ge4e227b3 2024-09-14T03:33:04.311 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:33:04.311 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:33:04.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:33:04.339 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-14T03:33:04.339 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:33:04.339 INFO:teuthology.packaging:ref: None 2024-09-14T03:33:04.339 INFO:teuthology.packaging:tag: None 2024-09-14T03:33:04.339 INFO:teuthology.packaging:branch: reef 2024-09-14T03:33:04.339 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:33:04.339 DEBUG:teuthology.orchestra.run.smithi001:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:33:04.350 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:33:04.350 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:33:04.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:33:04.378 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-14T03:33:04.378 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:33:04.378 INFO:teuthology.packaging:ref: None 2024-09-14T03:33:04.378 INFO:teuthology.packaging:tag: None 2024-09-14T03:33:04.378 INFO:teuthology.packaging:branch: reef 2024-09-14T03:33:04.378 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:33:04.379 DEBUG:teuthology.orchestra.run.smithi012:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:33:04.407 DEBUG:teuthology.orchestra.run.smithi001:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:33:04.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:33:04.485 DEBUG:teuthology.orchestra.run.smithi001:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:33:04.525 DEBUG:teuthology.orchestra.run.smithi012:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:33:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:check_obsoletes = 1 2024-09-14T03:33:04.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum clean all 2024-09-14T03:33:04.590 INFO:teuthology.orchestra.run.smithi012.stdout:check_obsoletes = 1 2024-09-14T03:33:04.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum clean all 2024-09-14T03:33:04.787 INFO:teuthology.orchestra.run.smithi001.stdout:45 files removed 2024-09-14T03:33:04.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-radosgw 2024-09-14T03:33:04.829 INFO:teuthology.orchestra.run.smithi012.stdout:45 files removed 2024-09-14T03:33:04.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-radosgw 2024-09-14T03:33:05.384 INFO:teuthology.orchestra.run.smithi001.stdout:ceph packages for x86_64 288 kB/s | 77 kB 00:00 2024-09-14T03:33:05.434 INFO:teuthology.orchestra.run.smithi012.stdout:ceph packages for x86_64 278 kB/s | 77 kB 00:00 2024-09-14T03:33:05.668 INFO:teuthology.orchestra.run.smithi001.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-09-14T03:33:05.727 INFO:teuthology.orchestra.run.smithi012.stdout:ceph noarch packages 46 kB/s | 11 kB 00:00 2024-09-14T03:33:06.119 INFO:teuthology.orchestra.run.smithi001.stdout:ceph source packages 4.6 kB/s | 1.9 kB 00:00 2024-09-14T03:33:06.303 INFO:teuthology.orchestra.run.smithi012.stdout:ceph source packages 3.5 kB/s | 1.9 kB 00:00 2024-09-14T03:33:06.635 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-14T03:33:08.227 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 4.4 MB/s | 8.2 MB 00:01 2024-09-14T03:33:08.728 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - AppStream 22 MB/s | 20 MB 00:00 2024-09-14T03:33:10.028 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-14T03:33:14.487 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - CRB 3.6 MB/s | 6.5 MB 00:01 2024-09-14T03:33:14.754 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.5 MB 00:00 2024-09-14T03:33:16.071 INFO:teuthology.orchestra.run.smithi001.stdout:CentOS Stream 9 - Extras packages 61 kB/s | 19 kB 00:00 2024-09-14T03:33:16.463 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 19 kB 00:00 2024-09-14T03:33:17.225 INFO:teuthology.orchestra.run.smithi001.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-14T03:33:17.433 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 27 MB/s | 23 MB 00:00 2024-09-14T03:33:22.848 INFO:teuthology.orchestra.run.smithi001.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-14T03:33:23.072 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 42 kB/s | 1.7 kB 00:00 2024-09-14T03:33:25.330 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:33:25.332 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:25.332 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:33:25.332 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-radosgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 7.7 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout:Upgrading: 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: librados2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.3 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: librbd1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-base x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 5.1 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 18 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-selinux x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 25 k 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T03:33:25.333 INFO:teuthology.orchestra.run.smithi001.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: libcephfs2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 712 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: libradosstriper1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 478 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: librgw2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.5 M 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ceph-argparse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 45 k 2024-09-14T03:33:25.334 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 129 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cephfs x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 162 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rados x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 321 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 300 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 99 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout:Install 28 Packages 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout:Upgrade 2 Packages 2024-09-14T03:33:25.335 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:25.336 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 53 M 2024-09-14T03:33:25.336 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:33:25.545 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 7.7 M 2024-09-14T03:33:25.547 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrading: 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: librados2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.3 M 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 5.1 M 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 18 M 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 25 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 712 k 2024-09-14T03:33:25.548 INFO:teuthology.orchestra.run.smithi012.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 478 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.5 M 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 45 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 129 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 162 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:33:25.549 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 321 k 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 300 k 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 99 k 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout:Install 28 Packages 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrade 2 Packages 2024-09-14T03:33:25.550 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:25.551 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 53 M 2024-09-14T03:33:25.551 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:33:26.073 INFO:teuthology.orchestra.run.smithi012.stdout:(1/30): ceph-base-18.2.4-845.ge4e227b3.el9.x86_ 12 MB/s | 5.1 MB 00:00 2024-09-14T03:33:26.149 INFO:teuthology.orchestra.run.smithi012.stdout:(2/30): ceph-selinux-18.2.4-845.ge4e227b3.el9.x 336 kB/s | 25 kB 00:00 2024-09-14T03:33:26.240 INFO:teuthology.orchestra.run.smithi012.stdout:(3/30): ceph-radosgw-18.2.4-845.ge4e227b3.el9.x 13 MB/s | 7.7 MB 00:00 2024-09-14T03:33:26.274 INFO:teuthology.orchestra.run.smithi012.stdout:(4/30): libcephfs2-18.2.4-845.ge4e227b3.el9.x86 5.6 MB/s | 712 kB 00:00 2024-09-14T03:33:26.299 INFO:teuthology.orchestra.run.smithi012.stdout:(5/30): libradosstriper1-18.2.4-845.ge4e227b3.e 8.0 MB/s | 478 kB 00:00 2024-09-14T03:33:26.383 INFO:teuthology.orchestra.run.smithi012.stdout:(6/30): python3-ceph-argparse-18.2.4-845.ge4e22 541 kB/s | 45 kB 00:00 2024-09-14T03:33:26.500 INFO:teuthology.orchestra.run.smithi012.stdout:(7/30): librgw2-18.2.4-845.ge4e227b3.el9.x86_64 20 MB/s | 4.5 MB 00:00 2024-09-14T03:33:26.525 INFO:teuthology.orchestra.run.smithi012.stdout:(8/30): python3-ceph-common-18.2.4-845.ge4e227b 910 kB/s | 129 kB 00:00 2024-09-14T03:33:26.559 INFO:teuthology.orchestra.run.smithi012.stdout:(9/30): python3-cephfs-18.2.4-845.ge4e227b3.el9 2.7 MB/s | 162 kB 00:00 2024-09-14T03:33:26.592 INFO:teuthology.orchestra.run.smithi012.stdout:(10/30): python3-rados-18.2.4-845.ge4e227b3.el9 4.7 MB/s | 321 kB 00:00 2024-09-14T03:33:26.626 INFO:teuthology.orchestra.run.smithi012.stdout:(11/30): python3-rbd-18.2.4-845.ge4e227b3.el9.x 4.4 MB/s | 300 kB 00:00 2024-09-14T03:33:26.659 INFO:teuthology.orchestra.run.smithi012.stdout:(12/30): python3-rgw-18.2.4-845.ge4e227b3.el9.x 1.5 MB/s | 99 kB 00:00 2024-09-14T03:33:26.791 INFO:teuthology.orchestra.run.smithi001.stdout:(1/30): ceph-base-18.2.4-845.ge4e227b3.el9.x86_ 4.8 MB/s | 5.1 MB 00:01 2024-09-14T03:33:26.841 INFO:teuthology.orchestra.run.smithi001.stdout:(2/30): ceph-selinux-18.2.4-845.ge4e227b3.el9.x 506 kB/s | 25 kB 00:00 2024-09-14T03:33:26.860 INFO:teuthology.orchestra.run.smithi012.stdout:(13/30): ceph-common-18.2.4-845.ge4e227b3.el9.x 15 MB/s | 18 MB 00:01 2024-09-14T03:33:26.902 INFO:teuthology.orchestra.run.smithi012.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 387 kB/s | 107 kB 00:00 2024-09-14T03:33:26.916 INFO:teuthology.orchestra.run.smithi001.stdout:(3/30): libcephfs2-18.2.4-845.ge4e227b3.el9.x86 9.3 MB/s | 712 kB 00:00 2024-09-14T03:33:26.960 INFO:teuthology.orchestra.run.smithi012.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 533 kB/s | 160 kB 00:00 2024-09-14T03:33:26.977 INFO:teuthology.orchestra.run.smithi012.stdout:(16/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 387 kB/s | 45 kB 00:00 2024-09-14T03:33:26.991 INFO:teuthology.orchestra.run.smithi001.stdout:(4/30): libradosstriper1-18.2.4-845.ge4e227b3.e 6.2 MB/s | 478 kB 00:00 2024-09-14T03:33:27.011 INFO:teuthology.orchestra.run.smithi012.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 1.2 MB/s | 42 kB 00:00 2024-09-14T03:33:27.045 INFO:teuthology.orchestra.run.smithi012.stdout:(18/30): librdkafka-1.6.1-102.el9.x86_64.rpm 4.5 MB/s | 662 kB 00:00 2024-09-14T03:33:27.070 INFO:teuthology.orchestra.run.smithi012.stdout:(19/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-14T03:33:27.120 INFO:teuthology.orchestra.run.smithi012.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 1.0 MB/s | 111 kB 00:00 2024-09-14T03:33:27.137 INFO:teuthology.orchestra.run.smithi012.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.7 MB/s | 26 kB 00:00 2024-09-14T03:33:27.154 INFO:teuthology.orchestra.run.smithi012.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 2.9 MB/s | 49 kB 00:00 2024-09-14T03:33:27.167 INFO:teuthology.orchestra.run.smithi001.stdout:(5/30): ceph-radosgw-18.2.4-845.ge4e227b3.el9.x 5.4 MB/s | 7.7 MB 00:01 2024-09-14T03:33:27.188 INFO:teuthology.orchestra.run.smithi012.stdout:(23/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.1 MB/s | 308 kB 00:00 2024-09-14T03:33:27.213 INFO:teuthology.orchestra.run.smithi012.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-09-14T03:33:27.225 INFO:teuthology.orchestra.run.smithi001.stdout:(6/30): python3-ceph-argparse-18.2.4-845.ge4e22 775 kB/s | 45 kB 00:00 2024-09-14T03:33:27.255 INFO:teuthology.orchestra.run.smithi012.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 4.5 MB/s | 191 kB 00:00 2024-09-14T03:33:27.284 INFO:teuthology.orchestra.run.smithi001.stdout:(7/30): python3-ceph-common-18.2.4-845.ge4e227b 2.2 MB/s | 129 kB 00:00 2024-09-14T03:33:27.297 INFO:teuthology.orchestra.run.smithi012.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-09-14T03:33:27.351 INFO:teuthology.orchestra.run.smithi001.stdout:(8/30): librgw2-18.2.4-845.ge4e227b3.el9.x86_64 12 MB/s | 4.5 MB 00:00 2024-09-14T03:33:27.376 INFO:teuthology.orchestra.run.smithi001.stdout:(9/30): python3-cephfs-18.2.4-845.ge4e227b3.el9 1.7 MB/s | 162 kB 00:00 2024-09-14T03:33:27.389 INFO:teuthology.orchestra.run.smithi012.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2024-09-14T03:33:27.410 INFO:teuthology.orchestra.run.smithi001.stdout:(10/30): python3-rados-18.2.4-845.ge4e227b3.el9 5.3 MB/s | 321 kB 00:00 2024-09-14T03:33:27.435 INFO:teuthology.orchestra.run.smithi001.stdout:(11/30): python3-rbd-18.2.4-845.ge4e227b3.el9.x 5.0 MB/s | 300 kB 00:00 2024-09-14T03:33:27.468 INFO:teuthology.orchestra.run.smithi001.stdout:(12/30): python3-rgw-18.2.4-845.ge4e227b3.el9.x 1.7 MB/s | 99 kB 00:00 2024-09-14T03:33:27.481 INFO:teuthology.orchestra.run.smithi012.stdout:(28/30): librados2-18.2.4-845.ge4e227b3.el9.x86 18 MB/s | 3.3 MB 00:00 2024-09-14T03:33:27.552 INFO:teuthology.orchestra.run.smithi001.stdout:(13/30): boost-program-options-1.75.0-8.el9.x86 914 kB/s | 107 kB 00:00 2024-09-14T03:33:27.556 INFO:teuthology.orchestra.run.smithi012.stdout:(29/30): libarrow-9.0.0-11.el9.x86_64.rpm 9.4 MB/s | 4.4 MB 00:00 2024-09-14T03:33:27.632 INFO:teuthology.orchestra.run.smithi012.stdout:(30/30): librbd1-18.2.4-845.ge4e227b3.el9.x86_6 12 MB/s | 3.0 MB 00:00 2024-09-14T03:33:27.637 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:27.637 INFO:teuthology.orchestra.run.smithi012.stdout:Total 25 MB/s | 53 MB 00:02 2024-09-14T03:33:27.712 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:33:27.736 INFO:teuthology.orchestra.run.smithi001.stdout:(14/30): ceph-common-18.2.4-845.ge4e227b3.el9.x 9.1 MB/s | 18 MB 00:02 2024-09-14T03:33:27.752 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:33:27.752 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:33:27.752 INFO:teuthology.orchestra.run.smithi001.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 226 kB/s | 45 kB 00:00 2024-09-14T03:33:27.778 INFO:teuthology.orchestra.run.smithi001.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 518 kB/s | 160 kB 00:00 2024-09-14T03:33:27.868 INFO:teuthology.orchestra.run.smithi001.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 468 kB/s | 42 kB 00:00 2024-09-14T03:33:27.920 INFO:teuthology.orchestra.run.smithi001.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.7 MB/s | 292 kB 00:00 2024-09-14T03:33:27.962 INFO:teuthology.orchestra.run.smithi001.stdout:(19/30): fmt-8.1.1-5.el9.x86_64.rpm 1.2 MB/s | 111 kB 00:00 2024-09-14T03:33:27.996 INFO:teuthology.orchestra.run.smithi001.stdout:(20/30): librdkafka-1.6.1-102.el9.x86_64.rpm 2.5 MB/s | 662 kB 00:00 2024-09-14T03:33:28.038 INFO:teuthology.orchestra.run.smithi001.stdout:(21/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.6 MB/s | 308 kB 00:00 2024-09-14T03:33:28.063 INFO:teuthology.orchestra.run.smithi001.stdout:(22/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 390 kB/s | 26 kB 00:00 2024-09-14T03:33:28.088 INFO:teuthology.orchestra.run.smithi001.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 968 kB/s | 49 kB 00:00 2024-09-14T03:33:28.113 INFO:teuthology.orchestra.run.smithi001.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-14T03:33:28.155 INFO:teuthology.orchestra.run.smithi001.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 4.5 MB/s | 191 kB 00:00 2024-09-14T03:33:28.198 INFO:teuthology.orchestra.run.smithi001.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-09-14T03:33:28.276 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:33:28.276 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:33:28.323 INFO:teuthology.orchestra.run.smithi001.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-09-14T03:33:28.381 INFO:teuthology.orchestra.run.smithi001.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 11 MB/s | 4.4 MB 00:00 2024-09-14T03:33:28.481 INFO:teuthology.orchestra.run.smithi001.stdout:(29/30): librados2-18.2.4-845.ge4e227b3.el9.x86 12 MB/s | 3.3 MB 00:00 2024-09-14T03:33:28.573 INFO:teuthology.orchestra.run.smithi001.stdout:(30/30): librbd1-18.2.4-845.ge4e227b3.el9.x86_6 12 MB/s | 3.0 MB 00:00 2024-09-14T03:33:28.578 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:28.579 INFO:teuthology.orchestra.run.smithi001.stdout:Total 16 MB/s | 53 MB 00:03 2024-09-14T03:33:28.651 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:33:28.690 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:33:28.690 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:33:29.176 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:33:29.177 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:33:29.779 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:33:29.865 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-14T03:33:29.895 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-14T03:33:29.946 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-14T03:33:30.354 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-14T03:33:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:30.540 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:33:30.581 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:33:30.611 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:30.623 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-14T03:33:30.654 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-14T03:33:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:30.704 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-14T03:33:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-14T03:33:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 9/32 2024-09-14T03:33:30.823 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:33:30.856 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:30.922 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:31.031 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-14T03:33:31.048 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-14T03:33:31.063 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-14T03:33:31.105 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-14T03:33:31.145 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-14T03:33:31.161 INFO:teuthology.orchestra.run.smithi001.stdout: Upgrading : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:31.171 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-14T03:33:31.208 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-14T03:33:31.238 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:31.254 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-14T03:33:31.323 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:33:31.362 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:31.408 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:31.437 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-14T03:33:31.481 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 9/32 2024-09-14T03:33:31.558 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:33:31.573 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-14T03:33:31.582 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:31.611 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:33:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:31.703 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:33:31.733 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 21/32 2024-09-14T03:33:31.758 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-14T03:33:31.769 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-14T03:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-14T03:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-14T03:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-14T03:33:31.889 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-14T03:33:31.925 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-14T03:33:31.958 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-14T03:33:32.196 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-14T03:33:32.239 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:33:32.274 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-14T03:33:32.310 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:33:32.312 INFO:teuthology.orchestra.run.smithi001.stdout: Upgrading : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:33:32.398 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 25/32 2024-09-14T03:33:32.404 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:33:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 26/32 2024-09-14T03:33:32.434 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 21/32 2024-09-14T03:33:32.470 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-14T03:33:32.900 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-14T03:33:32.923 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:33:33.137 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:33:33.225 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 25/32 2024-09-14T03:33:33.259 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 26/32 2024-09-14T03:33:34.572 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:34.612 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:35.190 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:35.240 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:33:35.340 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:33:35.341 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:33:35.341 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:33:35.341 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:35.393 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:35.430 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:35.996 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:33:36.042 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:33:36.144 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:33:36.144 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:33:36.144 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:33:36.144 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:36.185 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:46.036 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /sys 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /proc 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /mnt 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /var/tmp 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /home 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /root 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /tmp 2024-09-14T03:33:46.037 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /sys 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /proc 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /mnt 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /var/tmp 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /home 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /root 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout:skipping the directory /tmp 2024-09-14T03:33:46.445 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:33:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:33:46.842 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:33:46.842 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:33:46.842 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:33:46.843 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:33:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:46.844 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:33:46.934 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:33:46.934 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:47.343 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:33:47.370 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:47.373 INFO:teuthology.orchestra.run.smithi001.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:33:47.462 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:33:47.462 INFO:teuthology.orchestra.run.smithi001.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 1/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 3/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 4/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:48.045 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 8/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 9/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 12/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 13/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-14T03:33:48.046 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-14T03:33:48.047 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 31/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 1/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 3/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 4/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 8/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 9/32 2024-09-14T03:33:48.722 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 12/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 13/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-14T03:33:48.723 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-14T03:33:48.724 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-14T03:33:48.724 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-14T03:33:48.724 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-14T03:33:48.724 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-14T03:33:48.725 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:33:48.725 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-14T03:33:48.725 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 31/32 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout:Upgraded: 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.105 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:33:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.107 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.108 INFO:teuthology.orchestra.run.smithi012.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T03:33:49.108 INFO:teuthology.orchestra.run.smithi012.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:33:49.108 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:49.108 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:33:49.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-test 2024-09-14T03:33:49.719 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:33:49.719 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:49.719 INFO:teuthology.orchestra.run.smithi001.stdout:Upgraded: 2024-09-14T03:33:49.719 INFO:teuthology.orchestra.run.smithi001.stdout: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.719 INFO:teuthology.orchestra.run.smithi001.stdout: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.720 INFO:teuthology.orchestra.run.smithi001.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:33:49.721 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:49.722 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:33:50.048 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:27 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:33:50.128 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-test 2024-09-14T03:33:50.194 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:33:50.194 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:50.194 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 40 M 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 166 k 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout:Install 4 Packages 2024-09-14T03:33:50.195 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:50.196 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 40 M 2024-09-14T03:33:50.196 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 164 M 2024-09-14T03:33:50.196 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:33:50.383 INFO:teuthology.orchestra.run.smithi012.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.1 MB/s | 304 kB 00:00 2024-09-14T03:33:50.425 INFO:teuthology.orchestra.run.smithi012.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-14T03:33:50.492 INFO:teuthology.orchestra.run.smithi012.stdout:(3/4): libcephsqlite-18.2.4-845.ge4e227b3.el9.x 670 kB/s | 166 kB 00:00 2024-09-14T03:33:50.717 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:28 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:33:50.855 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-test x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 40 M 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: libcephsqlite x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 166 k 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:33:50.856 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:Install 4 Packages 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 40 M 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 164 M 2024-09-14T03:33:50.857 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:33:51.177 INFO:teuthology.orchestra.run.smithi001.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.5 MB/s | 304 kB 00:00 2024-09-14T03:33:51.227 INFO:teuthology.orchestra.run.smithi001.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-14T03:33:51.319 INFO:teuthology.orchestra.run.smithi001.stdout:(3/4): libcephsqlite-18.2.4-845.ge4e227b3.el9.x 498 kB/s | 166 kB 00:00 2024-09-14T03:33:51.584 INFO:teuthology.orchestra.run.smithi012.stdout:(4/4): ceph-test-18.2.4-845.ge4e227b3.el9.x86_6 30 MB/s | 40 MB 00:01 2024-09-14T03:33:51.586 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:51.586 INFO:teuthology.orchestra.run.smithi012.stdout:Total 29 MB/s | 40 MB 00:01 2024-09-14T03:33:51.615 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:33:51.637 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:33:51.637 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:33:51.982 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:33:51.982 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:33:52.403 INFO:teuthology.orchestra.run.smithi001.stdout:(4/4): ceph-test-18.2.4-845.ge4e227b3.el9.x86_6 28 MB/s | 40 MB 00:01 2024-09-14T03:33:52.405 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:33:52.405 INFO:teuthology.orchestra.run.smithi001.stdout:Total 26 MB/s | 40 MB 00:01 2024-09-14T03:33:52.434 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:33:52.456 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:33:52.456 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:33:52.482 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:33:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T03:33:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T03:33:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:33:52.805 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:33:52.805 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:33:53.296 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:33:53.370 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T03:33:53.412 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T03:33:53.438 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:33:56.560 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:33:56.653 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:33:57.354 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:33:57.431 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:33:58.788 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:33:58.788 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 1/4 2024-09-14T03:33:58.788 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2/4 2024-09-14T03:33:58.788 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T03:33:59.090 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T03:33:59.090 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:59.090 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:33:59.091 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:33:59.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph 2024-09-14T03:33:59.414 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:33:59.414 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 1/4 2024-09-14T03:33:59.414 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2/4 2024-09-14T03:33:59.414 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:33:59.765 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:33:59.994 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:36 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:00.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph 2024-09-14T03:34:00.145 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:00.147 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 6.5 k 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 2.1 M 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 1.5 M 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 246 k 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.7 M 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 17 M 2024-09-14T03:34:00.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:34:00.149 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:34:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:Install 33 Packages 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 29 M 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 104 M 2024-09-14T03:34:00.151 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:00.401 INFO:teuthology.orchestra.run.smithi012.stdout:(1/33): ceph-18.2.4-845.ge4e227b3.el9.x86_64.rp 54 kB/s | 6.5 kB 00:00 2024-09-14T03:34:00.576 INFO:teuthology.orchestra.run.smithi012.stdout:(2/33): ceph-mgr-18.2.4-845.ge4e227b3.el9.x86_6 4.9 MB/s | 1.5 MB 00:00 2024-09-14T03:34:00.635 INFO:teuthology.orchestra.run.smithi012.stdout:(3/33): ceph-mds-18.2.4-845.ge4e227b3.el9.x86_6 5.9 MB/s | 2.1 MB 00:00 2024-09-14T03:34:00.666 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:38 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:00.735 INFO:teuthology.orchestra.run.smithi012.stdout:(4/33): ceph-mon-18.2.4-845.ge4e227b3.el9.x86_6 14 MB/s | 4.7 MB 00:00 2024-09-14T03:34:00.760 INFO:teuthology.orchestra.run.smithi012.stdout:(5/33): ceph-mgr-modules-core-18.2.4-845.ge4e22 1.9 MB/s | 246 kB 00:00 2024-09-14T03:34:00.807 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout: ceph x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 6.5 k 2024-09-14T03:34:00.809 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mds x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 2.1 M 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 1.5 M 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-modules-core noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 246 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mon x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.7 M 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-osd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 17 M 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:34:00.810 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:34:00.811 INFO:teuthology.orchestra.run.smithi001.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout:Install 33 Packages 2024-09-14T03:34:00.812 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:00.813 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 29 M 2024-09-14T03:34:00.813 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 104 M 2024-09-14T03:34:00.813 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:00.919 INFO:teuthology.orchestra.run.smithi012.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.3 MB/s | 253 kB 00:00 2024-09-14T03:34:00.969 INFO:teuthology.orchestra.run.smithi012.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-14T03:34:01.136 INFO:teuthology.orchestra.run.smithi012.stdout:(8/33): ceph-osd-18.2.4-845.ge4e227b3.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-14T03:34:01.162 INFO:teuthology.orchestra.run.smithi012.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 704 kB/s | 135 kB 00:00 2024-09-14T03:34:01.204 INFO:teuthology.orchestra.run.smithi012.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 2.8 MB/s | 1.2 MB 00:00 2024-09-14T03:34:01.229 INFO:teuthology.orchestra.run.smithi012.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.1 MB/s | 215 kB 00:00 2024-09-14T03:34:01.288 INFO:teuthology.orchestra.run.smithi012.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 829 kB/s | 125 kB 00:00 2024-09-14T03:34:01.579 INFO:teuthology.orchestra.run.smithi001.stdout:(1/33): ceph-18.2.4-845.ge4e227b3.el9.x86_64.rp 50 kB/s | 6.5 kB 00:00 2024-09-14T03:34:01.754 INFO:teuthology.orchestra.run.smithi001.stdout:(2/33): ceph-mgr-18.2.4-845.ge4e227b3.el9.x86_6 4.8 MB/s | 1.5 MB 00:00 2024-09-14T03:34:01.796 INFO:teuthology.orchestra.run.smithi012.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 61 kB/s | 35 kB 00:00 2024-09-14T03:34:01.880 INFO:teuthology.orchestra.run.smithi012.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 86 kB/s | 51 kB 00:00 2024-09-14T03:34:01.930 INFO:teuthology.orchestra.run.smithi012.stdout:(15/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 878 kB/s | 43 kB 00:00 2024-09-14T03:34:01.955 INFO:teuthology.orchestra.run.smithi012.stdout:(16/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 6.8 MB/s | 172 kB 00:00 2024-09-14T03:34:01.971 INFO:teuthology.orchestra.run.smithi001.stdout:(3/33): ceph-mon-18.2.4-845.ge4e227b3.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-09-14T03:34:01.989 INFO:teuthology.orchestra.run.smithi012.stdout:(17/33): python3-mako-1.1.4-6.el9.noarch.rpm 219 kB/s | 172 kB 00:00 2024-09-14T03:34:02.021 INFO:teuthology.orchestra.run.smithi001.stdout:(4/33): ceph-mgr-modules-core-18.2.4-845.ge4e22 4.8 MB/s | 246 kB 00:00 2024-09-14T03:34:02.022 INFO:teuthology.orchestra.run.smithi012.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 5.3 MB/s | 358 kB 00:00 2024-09-14T03:34:02.064 INFO:teuthology.orchestra.run.smithi012.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 527 kB/s | 18 kB 00:00 2024-09-14T03:34:02.089 INFO:teuthology.orchestra.run.smithi012.stdout:(20/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 107 kB/s | 11 kB 00:00 2024-09-14T03:34:02.115 INFO:teuthology.orchestra.run.smithi012.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 462 kB/s | 23 kB 00:00 2024-09-14T03:34:02.140 INFO:teuthology.orchestra.run.smithi012.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-09-14T03:34:02.168 INFO:teuthology.orchestra.run.smithi012.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 399 kB/s | 20 kB 00:00 2024-09-14T03:34:02.191 INFO:teuthology.orchestra.run.smithi012.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-09-14T03:34:02.216 INFO:teuthology.orchestra.run.smithi012.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-14T03:34:02.241 INFO:teuthology.orchestra.run.smithi012.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 663 kB/s | 16 kB 00:00 2024-09-14T03:34:02.266 INFO:teuthology.orchestra.run.smithi012.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 3.5 MB/s | 272 kB 00:00 2024-09-14T03:34:02.292 INFO:teuthology.orchestra.run.smithi012.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-14T03:34:02.313 INFO:teuthology.orchestra.run.smithi001.stdout:(5/33): ceph-osd-18.2.4-845.ge4e227b3.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-14T03:34:02.317 INFO:teuthology.orchestra.run.smithi012.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 714 kB/s | 36 kB 00:00 2024-09-14T03:34:02.338 INFO:teuthology.orchestra.run.smithi001.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 799 kB/s | 253 kB 00:00 2024-09-14T03:34:02.342 INFO:teuthology.orchestra.run.smithi012.stdout:(30/33): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-14T03:34:02.367 INFO:teuthology.orchestra.run.smithi012.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 806 kB/s | 20 kB 00:00 2024-09-14T03:34:02.393 INFO:teuthology.orchestra.run.smithi012.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.5 MB/s | 427 kB 00:00 2024-09-14T03:34:02.397 INFO:teuthology.orchestra.run.smithi001.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 1.8 MB/s | 106 kB 00:00 2024-09-14T03:34:02.418 INFO:teuthology.orchestra.run.smithi012.stdout:(33/33): python3-toml-0.10.2-6.el9.noarch.rpm 67 kB/s | 42 kB 00:00 2024-09-14T03:34:02.423 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:02.423 INFO:teuthology.orchestra.run.smithi012.stdout:Total 13 MB/s | 29 MB 00:02 2024-09-14T03:34:02.456 INFO:teuthology.orchestra.run.smithi001.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.3 MB/s | 135 kB 00:00 2024-09-14T03:34:02.514 INFO:teuthology.orchestra.run.smithi001.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.1 MB/s | 125 kB 00:00 2024-09-14T03:34:02.550 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:02.556 INFO:teuthology.orchestra.run.smithi001.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 5.1 MB/s | 1.2 MB 00:00 2024-09-14T03:34:02.581 INFO:teuthology.orchestra.run.smithi001.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.1 MB/s | 215 kB 00:00 2024-09-14T03:34:02.592 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:02.592 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:02.649 INFO:teuthology.orchestra.run.smithi001.stdout:(12/33): python3-markupsafe-1.1.1-12.el9.x86_64 521 kB/s | 35 kB 00:00 2024-09-14T03:34:02.674 INFO:teuthology.orchestra.run.smithi001.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.4 MB/s | 172 kB 00:00 2024-09-14T03:34:02.699 INFO:teuthology.orchestra.run.smithi001.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-09-14T03:34:02.724 INFO:teuthology.orchestra.run.smithi001.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 830 kB/s | 42 kB 00:00 2024-09-14T03:34:02.766 INFO:teuthology.orchestra.run.smithi001.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 650 kB/s | 43 kB 00:00 2024-09-14T03:34:02.800 INFO:teuthology.orchestra.run.smithi001.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.2 MB/s | 172 kB 00:00 2024-09-14T03:34:02.825 INFO:teuthology.orchestra.run.smithi001.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 6.2 MB/s | 358 kB 00:00 2024-09-14T03:34:02.850 INFO:teuthology.orchestra.run.smithi001.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 212 kB/s | 11 kB 00:00 2024-09-14T03:34:02.876 INFO:teuthology.orchestra.run.smithi001.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 351 kB/s | 18 kB 00:00 2024-09-14T03:34:02.901 INFO:teuthology.orchestra.run.smithi001.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 463 kB/s | 23 kB 00:00 2024-09-14T03:34:02.926 INFO:teuthology.orchestra.run.smithi001.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-09-14T03:34:02.960 INFO:teuthology.orchestra.run.smithi001.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 336 kB/s | 20 kB 00:00 2024-09-14T03:34:02.985 INFO:teuthology.orchestra.run.smithi001.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 792 kB/s | 46 kB 00:00 2024-09-14T03:34:03.000 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:03.001 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:03.010 INFO:teuthology.orchestra.run.smithi001.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-14T03:34:03.036 INFO:teuthology.orchestra.run.smithi001.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 662 kB/s | 16 kB 00:00 2024-09-14T03:34:03.061 INFO:teuthology.orchestra.run.smithi001.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 3.5 MB/s | 272 kB 00:00 2024-09-14T03:34:03.086 INFO:teuthology.orchestra.run.smithi001.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-09-14T03:34:03.111 INFO:teuthology.orchestra.run.smithi001.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 717 kB/s | 36 kB 00:00 2024-09-14T03:34:03.136 INFO:teuthology.orchestra.run.smithi001.stdout:(30/33): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-14T03:34:03.162 INFO:teuthology.orchestra.run.smithi001.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 797 kB/s | 20 kB 00:00 2024-09-14T03:34:03.187 INFO:teuthology.orchestra.run.smithi001.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.5 MB/s | 427 kB 00:00 2024-09-14T03:34:03.820 INFO:teuthology.orchestra.run.smithi001.stdout:(33/33): ceph-mds-18.2.4-845.ge4e227b3.el9.x86_ 905 kB/s | 2.1 MB 00:02 2024-09-14T03:34:03.824 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:03.824 INFO:teuthology.orchestra.run.smithi001.stdout:Total 9.7 MB/s | 29 MB 00:03 2024-09-14T03:34:03.952 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:03.993 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:03.993 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:04.219 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:04.270 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-14T03:34:04.306 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-14T03:34:04.338 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-14T03:34:04.387 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:04.388 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:04.435 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-14T03:34:04.528 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-14T03:34:04.583 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-14T03:34:04.620 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-14T03:34:04.647 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-14T03:34:04.681 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-14T03:34:04.715 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-14T03:34:04.756 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-14T03:34:04.799 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-14T03:34:04.833 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-14T03:34:04.880 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-14T03:34:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-14T03:34:05.126 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-14T03:34:05.208 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-14T03:34:05.267 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-14T03:34:05.305 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-14T03:34:05.363 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-14T03:34:05.436 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-14T03:34:05.625 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:05.633 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-14T03:34:05.683 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-14T03:34:05.707 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-14T03:34:05.740 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-14T03:34:05.773 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-14T03:34:05.782 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-14T03:34:05.868 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-14T03:34:05.879 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-14T03:34:05.915 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-14T03:34:05.944 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-14T03:34:05.992 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-14T03:34:06.026 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-14T03:34:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-14T03:34:06.056 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-14T03:34:06.082 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-14T03:34:06.115 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-14T03:34:06.157 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-14T03:34:06.200 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-14T03:34:06.214 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 28/33 2024-09-14T03:34:06.234 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-14T03:34:06.250 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:34:06.272 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-14T03:34:06.277 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:34:06.278 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:06.278 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:34:06.278 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:34:06.278 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:34:06.278 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:06.349 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-14T03:34:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-14T03:34:06.583 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-14T03:34:06.643 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-14T03:34:06.681 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-14T03:34:06.739 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-14T03:34:06.802 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-14T03:34:06.997 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-14T03:34:07.051 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-14T03:34:07.132 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-14T03:34:07.232 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-14T03:34:07.267 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-14T03:34:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-14T03:34:07.548 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 28/33 2024-09-14T03:34:07.576 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:34:07.604 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:07.785 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:34:07.817 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:34:07.817 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:07.817 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:34:07.817 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:34:07.817 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:34:07.818 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:08.311 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:34:08.338 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:34:08.339 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:08.339 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:34:08.339 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:34:08.339 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:34:08.340 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:34:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:34:09.120 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:34:09.150 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:34:09.150 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:09.150 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:34:09.150 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:34:09.151 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:34:09.151 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:09.621 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:34:09.646 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:34:09.646 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:09.646 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:34:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:34:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:34:09.647 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:09.863 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:34:09.889 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:09.954 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 1/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 3/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 4/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 5/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 6/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-14T03:34:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-14T03:34:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-14T03:34:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-14T03:34:11.341 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:11.342 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:34:11.343 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:34:11.344 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:11.345 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:11.624 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 1/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 3/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 4/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 5/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 6/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-14T03:34:11.625 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-14T03:34:11.626 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-14T03:34:11.627 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-14T03:34:11.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-base 2024-09-14T03:34:12.315 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:49 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:12.409 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:12.454 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:12.455 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:12.455 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:12.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install cephadm 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:34:12.515 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:34:12.516 INFO:teuthology.orchestra.run.smithi001.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:34:12.517 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:12.518 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:12.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-base 2024-09-14T03:34:13.104 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:50 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:13.242 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 226 k 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:34:13.243 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:13.244 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 226 k 2024-09-14T03:34:13.244 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 217 k 2024-09-14T03:34:13.244 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:13.446 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:51 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:13.502 INFO:teuthology.orchestra.run.smithi012.stdout:cephadm-18.2.4-845.ge4e227b3.el9.noarch.rpm 874 kB/s | 226 kB 00:00 2024-09-14T03:34:13.503 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:13.503 INFO:teuthology.orchestra.run.smithi012.stdout:Total 869 kB/s | 226 kB 00:00 2024-09-14T03:34:13.503 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:13.509 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:13.509 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:13.512 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:13.512 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:13.541 INFO:teuthology.orchestra.run.smithi001.stdout:Package ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:13.588 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:13.588 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:13.589 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:13.589 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:13.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install cephadm 2024-09-14T03:34:14.168 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:14.240 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:52 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:14.251 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:14.377 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:14.377 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout: cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 226 k 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:34:14.378 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:14.379 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 226 k 2024-09-14T03:34:14.379 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 217 k 2024-09-14T03:34:14.379 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:14.606 INFO:teuthology.orchestra.run.smithi001.stdout:cephadm-18.2.4-845.ge4e227b3.el9.noarch.rpm 997 kB/s | 226 kB 00:00 2024-09-14T03:34:14.606 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:14.606 INFO:teuthology.orchestra.run.smithi001.stdout:Total 989 kB/s | 226 kB 00:00 2024-09-14T03:34:14.607 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:14.612 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:14.612 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:14.615 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:14.616 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:14.687 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:14.879 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:14.879 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:14.879 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:14.880 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:14.880 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:14.880 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:15.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T03:34:15.111 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:15.188 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:15.507 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:15.668 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:52 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:15.773 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:34:15.773 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:15.774 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:15.774 INFO:teuthology.orchestra.run.smithi001.stdout: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:15.774 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:15.774 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:15.808 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:15.809 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:15.809 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-09-14T03:34:15.809 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:15.809 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 143 k 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:34:15.810 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:15.811 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 143 k 2024-09-14T03:34:15.811 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 395 k 2024-09-14T03:34:15.811 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:15.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T03:34:16.092 INFO:teuthology.orchestra.run.smithi012.stdout:ceph-immutable-object-cache-18.2.4-845.ge4e227b 505 kB/s | 143 kB 00:00 2024-09-14T03:34:16.092 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:16.093 INFO:teuthology.orchestra.run.smithi012.stdout:Total 502 kB/s | 143 kB 00:00 2024-09-14T03:34:16.093 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:16.101 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:16.101 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:16.162 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:16.162 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:16.437 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:16.501 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:16.527 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:16.527 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:16.528 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:34:16.528 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:16.556 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:54 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:16.694 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repo Size 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 143 k 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 143 k 2024-09-14T03:34:16.695 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 395 k 2024-09-14T03:34:16.696 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:16.903 INFO:teuthology.orchestra.run.smithi001.stdout:ceph-immutable-object-cache-18.2.4-845.ge4e227b 688 kB/s | 143 kB 00:00 2024-09-14T03:34:16.904 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:16.904 INFO:teuthology.orchestra.run.smithi001.stdout:Total 683 kB/s | 143 kB 00:00 2024-09-14T03:34:16.904 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:16.912 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:16.912 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:16.973 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:16.973 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:17.127 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:17.179 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:17.206 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:17.207 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:34:17.207 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:34:17.207 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:17.532 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:17.532 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:17.532 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:17.532 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:17.533 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:17.533 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:17.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr 2024-09-14T03:34:18.155 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:34:18.155 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:18.155 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:18.156 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:18.156 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:18.156 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:18.323 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:55 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:18.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-mgr 2024-09-14T03:34:18.418 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:18.462 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:18.463 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:18.463 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:18.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T03:34:18.973 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:19.067 INFO:teuthology.orchestra.run.smithi001.stdout:Package ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:19.111 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:19.112 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:19.113 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:19.118 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:19.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T03:34:19.256 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:19.256 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:19.256 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 3.5 M 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 24 k 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 16 k 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout:Install 5 Packages 2024-09-14T03:34:19.257 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:19.258 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.8 M 2024-09-14T03:34:19.258 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 87 M 2024-09-14T03:34:19.258 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:19.478 INFO:teuthology.orchestra.run.smithi012.stdout:(1/5): ceph-prometheus-alerts-18.2.4-845.ge4e22 102 kB/s | 16 kB 00:00 2024-09-14T03:34:19.503 INFO:teuthology.orchestra.run.smithi012.stdout:(2/5): ceph-grafana-dashboards-18.2.4-845.ge4e2 131 kB/s | 24 kB 00:00 2024-09-14T03:34:19.545 INFO:teuthology.orchestra.run.smithi012.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 462 kB/s | 31 kB 00:00 2024-09-14T03:34:19.579 INFO:teuthology.orchestra.run.smithi012.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.4 MB/s | 188 kB 00:00 2024-09-14T03:34:19.687 INFO:teuthology.orchestra.run.smithi012.stdout:(5/5): ceph-mgr-dashboard-18.2.4-845.ge4e227b3. 9.5 MB/s | 3.5 MB 00:00 2024-09-14T03:34:19.691 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:19.691 INFO:teuthology.orchestra.run.smithi012.stdout:Total 8.7 MB/s | 3.8 MB 00:00 2024-09-14T03:34:19.691 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:19.699 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:19.699 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:19.760 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:00:57 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:19.790 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:19.791 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:19.898 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:19.898 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:19.898 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-dashboard noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 3.5 M 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-grafana-dashboards noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 24 k 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-prometheus-alerts noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 16 k 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:19.899 INFO:teuthology.orchestra.run.smithi001.stdout:Install 5 Packages 2024-09-14T03:34:19.900 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:19.900 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 3.8 M 2024-09-14T03:34:19.900 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 87 M 2024-09-14T03:34:19.900 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:20.124 INFO:teuthology.orchestra.run.smithi001.stdout:(1/5): ceph-prometheus-alerts-18.2.4-845.ge4e22 100 kB/s | 16 kB 00:00 2024-09-14T03:34:20.136 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-14T03:34:20.149 INFO:teuthology.orchestra.run.smithi001.stdout:(2/5): ceph-grafana-dashboards-18.2.4-845.ge4e2 129 kB/s | 24 kB 00:00 2024-09-14T03:34:20.176 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-14T03:34:20.191 INFO:teuthology.orchestra.run.smithi001.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 460 kB/s | 31 kB 00:00 2024-09-14T03:34:20.213 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:34:20.216 INFO:teuthology.orchestra.run.smithi001.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-09-14T03:34:20.342 INFO:teuthology.orchestra.run.smithi001.stdout:(5/5): ceph-mgr-dashboard-18.2.4-845.ge4e227b3. 9.3 MB/s | 3.5 MB 00:00 2024-09-14T03:34:20.345 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:20.349 INFO:teuthology.orchestra.run.smithi001.stdout:Total 8.5 MB/s | 3.8 MB 00:00 2024-09-14T03:34:20.349 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:20.353 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:20.354 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:20.456 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:20.456 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-14T03:34:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-14T03:34:20.909 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:34:21.332 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 4/5 2024-09-14T03:34:21.368 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:34:22.026 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 4/5 2024-09-14T03:34:22.062 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:34:22.448 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:34:22.448 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 1/5 2024-09-14T03:34:22.448 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2/5 2024-09-14T03:34:22.448 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:34:22.448 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-14T03:34:22.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-14T03:34:22.739 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:22.739 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:22.739 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:22.739 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:34:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:34:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:22.740 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:22.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T03:34:23.092 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:34:23.092 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 1/5 2024-09-14T03:34:23.092 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2/5 2024-09-14T03:34:23.092 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:34:23.092 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-14T03:34:23.363 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-14T03:34:23.363 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:23.363 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:23.363 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:23.363 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:23.364 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:23.364 INFO:teuthology.orchestra.run.smithi001.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:34:23.364 INFO:teuthology.orchestra.run.smithi001.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:34:23.364 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:23.364 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:23.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T03:34:23.562 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:00 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:23.701 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:23.702 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-09-14T03:34:23.702 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:23.702 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 7.4 M 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:34:23.703 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout:Install 12 Packages 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 38 M 2024-09-14T03:34:23.704 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 212 M 2024-09-14T03:34:23.705 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:24.142 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:02 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:24.203 INFO:teuthology.orchestra.run.smithi012.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.6 MB/s | 191 kB 00:00 2024-09-14T03:34:24.261 INFO:teuthology.orchestra.run.smithi012.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 4.4 MB/s | 801 kB 00:00 2024-09-14T03:34:24.280 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:24.281 INFO:teuthology.orchestra.run.smithi001.stdout:====================================================================================== 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout:====================================================================================== 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 7.4 M 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:34:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout:====================================================================================== 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout:Install 12 Packages 2024-09-14T03:34:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:24.284 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 38 M 2024-09-14T03:34:24.284 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 212 M 2024-09-14T03:34:24.284 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:24.512 INFO:teuthology.orchestra.run.smithi012.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 106 kB/s | 33 kB 00:00 2024-09-14T03:34:24.604 INFO:teuthology.orchestra.run.smithi012.stdout:(4/12): ceph-mgr-diskprediction-local-18.2.4-84 14 MB/s | 7.4 MB 00:00 2024-09-14T03:34:24.629 INFO:teuthology.orchestra.run.smithi012.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 156 kB/s | 18 kB 00:00 2024-09-14T03:34:24.904 INFO:teuthology.orchestra.run.smithi012.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 128 kB/s | 38 kB 00:00 2024-09-14T03:34:24.957 INFO:teuthology.orchestra.run.smithi001.stdout:(1/12): ceph-mgr-diskprediction-local-18.2.4-84 16 MB/s | 7.4 MB 00:00 2024-09-14T03:34:25.040 INFO:teuthology.orchestra.run.smithi001.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 394 kB/s | 33 kB 00:00 2024-09-14T03:34:25.216 INFO:teuthology.orchestra.run.smithi001.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 17 MB/s | 3.0 MB 00:00 2024-09-14T03:34:25.257 INFO:teuthology.orchestra.run.smithi001.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 438 kB/s | 18 kB 00:00 2024-09-14T03:34:25.279 INFO:teuthology.orchestra.run.smithi012.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 652 kB/s | 245 kB 00:00 2024-09-14T03:34:25.299 INFO:teuthology.orchestra.run.smithi001.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 925 kB/s | 38 kB 00:00 2024-09-14T03:34:25.350 INFO:teuthology.orchestra.run.smithi001.stdout:(6/12): libquadmath-11.5.0-2.el9.x86_64.rpm 225 kB/s | 191 kB 00:00 2024-09-14T03:34:25.434 INFO:teuthology.orchestra.run.smithi001.stdout:(7/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 37 MB/s | 4.9 MB 00:00 2024-09-14T03:34:25.501 INFO:teuthology.orchestra.run.smithi001.stdout:(8/12): libgfortran-11.5.0-2.el9.x86_64.rpm 803 kB/s | 801 kB 00:00 2024-09-14T03:34:25.576 INFO:teuthology.orchestra.run.smithi001.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 36 MB/s | 5.1 MB 00:00 2024-09-14T03:34:25.596 INFO:teuthology.orchestra.run.smithi012.stdout:(8/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 5.1 MB/s | 4.9 MB 00:00 2024-09-14T03:34:25.601 INFO:teuthology.orchestra.run.smithi001.stdout:(10/12): python3-devel-3.9.19-8.el9.x86_64.rpm 977 kB/s | 245 kB 00:00 2024-09-14T03:34:25.701 INFO:teuthology.orchestra.run.smithi001.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.2 MB/s | 250 kB 00:00 2024-09-14T03:34:25.713 INFO:teuthology.orchestra.run.smithi012.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 250 kB 00:00 2024-09-14T03:34:25.876 INFO:teuthology.orchestra.run.smithi001.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 54 MB/s | 16 MB 00:00 2024-09-14T03:34:25.877 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:25.878 INFO:teuthology.orchestra.run.smithi001.stdout:Total 24 MB/s | 38 MB 00:01 2024-09-14T03:34:25.930 INFO:teuthology.orchestra.run.smithi012.stdout:(10/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rp 1.8 MB/s | 3.0 MB 00:01 2024-09-14T03:34:26.248 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:26.276 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:26.276 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:26.580 INFO:teuthology.orchestra.run.smithi012.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 19 MB/s | 16 MB 00:00 2024-09-14T03:34:26.788 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:26.788 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:27.438 INFO:teuthology.orchestra.run.smithi012.stdout:(12/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.4 MB/s | 5.1 MB 00:02 2024-09-14T03:34:27.439 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:27.440 INFO:teuthology.orchestra.run.smithi012.stdout:Total 10 MB/s | 38 MB 00:03 2024-09-14T03:34:27.574 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:27.633 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T03:34:27.675 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:34:27.779 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:34:27.811 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:27.826 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T03:34:27.838 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:27.838 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:28.017 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T03:34:28.049 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T03:34:28.128 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T03:34:28.366 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:28.366 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T03:34:28.958 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T03:34:29.158 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:29.235 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T03:34:29.279 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:34:29.392 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:34:29.430 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T03:34:29.626 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T03:34:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T03:34:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T03:34:30.289 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:34:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T03:34:30.571 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T03:34:31.377 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T03:34:31.407 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:34:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:34:32.981 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T03:34:33.010 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:34:33.964 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:34:33.964 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 1/12 2024-09-14T03:34:33.964 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:34:33.964 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:34:33.964 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:34:33.965 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:34:34.421 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:34:34.422 INFO:teuthology.orchestra.run.smithi001.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:34:34.423 INFO:teuthology.orchestra.run.smithi001.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:34:34.423 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:34.423 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:34.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-mgr-rook 2024-09-14T03:34:35.354 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:13 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:35.495 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:35.496 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:35.496 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:35.496 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:35.496 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-rook noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 49 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:34:35.497 INFO:teuthology.orchestra.run.smithi001.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout:Install 13 Packages 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 2.2 M 2024-09-14T03:34:35.498 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 26 M 2024-09-14T03:34:35.499 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:35.565 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 1/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:34:35.566 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T03:34:35.748 INFO:teuthology.orchestra.run.smithi001.stdout:(1/13): ceph-mgr-rook-18.2.4-845.ge4e227b3.el9. 286 kB/s | 49 kB 00:00 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:34:35.985 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:35.986 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:36.107 INFO:teuthology.orchestra.run.smithi001.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 36 kB/s | 19 kB 00:00 2024-09-14T03:34:36.140 INFO:teuthology.orchestra.run.smithi001.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 45 kB/s | 26 kB 00:00 2024-09-14T03:34:36.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-rook 2024-09-14T03:34:36.491 INFO:teuthology.orchestra.run.smithi001.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 299 kB/s | 222 kB 00:00 2024-09-14T03:34:36.841 INFO:teuthology.orchestra.run.smithi001.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 216 kB/s | 159 kB 00:00 2024-09-14T03:34:36.883 INFO:teuthology.orchestra.run.smithi001.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 780 kB/s | 32 kB 00:00 2024-09-14T03:34:36.908 INFO:teuthology.orchestra.run.smithi001.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 363 kB/s | 279 kB 00:00 2024-09-14T03:34:36.925 INFO:teuthology.orchestra.run.smithi001.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 337 kB/s | 14 kB 00:00 2024-09-14T03:34:36.931 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:13 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:36.959 INFO:teuthology.orchestra.run.smithi001.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 4.3 MB/s | 220 kB 00:00 2024-09-14T03:34:36.975 INFO:teuthology.orchestra.run.smithi001.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 4.0 MB/s | 59 kB 00:00 2024-09-14T03:34:37.001 INFO:teuthology.orchestra.run.smithi001.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.5 MB/s | 90 kB 00:00 2024-09-14T03:34:37.043 INFO:teuthology.orchestra.run.smithi001.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 8.7 MB/s | 1.0 MB 00:00 2024-09-14T03:34:37.076 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:37.077 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 49 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:34:37.078 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout:Install 13 Packages 2024-09-14T03:34:37.079 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:37.080 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 2.2 M 2024-09-14T03:34:37.080 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 26 M 2024-09-14T03:34:37.080 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:37.084 INFO:teuthology.orchestra.run.smithi001.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 90 kB/s | 54 kB 00:00 2024-09-14T03:34:37.088 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:37.088 INFO:teuthology.orchestra.run.smithi001.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-09-14T03:34:37.161 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:37.179 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:37.179 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:37.278 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:37.279 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:37.360 INFO:teuthology.orchestra.run.smithi012.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 228 kB/s | 19 kB 00:00 2024-09-14T03:34:37.385 INFO:teuthology.orchestra.run.smithi012.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 235 kB/s | 26 kB 00:00 2024-09-14T03:34:37.452 INFO:teuthology.orchestra.run.smithi012.stdout:(3/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.3 MB/s | 159 kB 00:00 2024-09-14T03:34:37.478 INFO:teuthology.orchestra.run.smithi012.stdout:(4/13): ceph-mgr-rook-18.2.4-845.ge4e227b3.el9. 244 kB/s | 49 kB 00:00 2024-09-14T03:34:37.503 INFO:teuthology.orchestra.run.smithi012.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-09-14T03:34:37.528 INFO:teuthology.orchestra.run.smithi012.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.6 MB/s | 279 kB 00:00 2024-09-14T03:34:37.570 INFO:teuthology.orchestra.run.smithi012.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 481 kB/s | 32 kB 00:00 2024-09-14T03:34:37.595 INFO:teuthology.orchestra.run.smithi012.stdout:(8/13): python3-requests-oauthlib-1.3.0-12.el9. 456 kB/s | 54 kB 00:00 2024-09-14T03:34:37.621 INFO:teuthology.orchestra.run.smithi012.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 166 kB/s | 14 kB 00:00 2024-09-14T03:34:37.655 INFO:teuthology.orchestra.run.smithi012.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.2 MB/s | 59 kB 00:00 2024-09-14T03:34:37.688 INFO:teuthology.orchestra.run.smithi012.stdout:(11/13): python3-google-auth-2.34.0-1.el9.noarc 1.9 MB/s | 220 kB 00:00 2024-09-14T03:34:37.713 INFO:teuthology.orchestra.run.smithi012.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.5 MB/s | 90 kB 00:00 2024-09-14T03:34:37.772 INFO:teuthology.orchestra.run.smithi012.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 5.8 MB/s | 1.0 MB 00:00 2024-09-14T03:34:37.774 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:37.775 INFO:teuthology.orchestra.run.smithi012.stdout:Total 3.2 MB/s | 2.2 MB 00:00 2024-09-14T03:34:37.803 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:37.842 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:37.859 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:37.859 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:37.955 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:37.955 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:37.977 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T03:34:38.029 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T03:34:38.086 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T03:34:38.120 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T03:34:38.163 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T03:34:38.270 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T03:34:38.379 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T03:34:38.431 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T03:34:38.496 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:38.685 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T03:34:38.758 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T03:34:38.814 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T03:34:38.849 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T03:34:38.892 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T03:34:38.998 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T03:34:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T03:34:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T03:34:39.190 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T03:34:39.235 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T03:34:39.269 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T03:34:39.318 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T03:34:39.347 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:34:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T03:34:39.973 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T03:34:40.007 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T03:34:40.056 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T03:34:40.085 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 1/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T03:34:40.210 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T03:34:40.211 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T03:34:40.211 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:34:40.545 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:40.546 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:40.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T03:34:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 1/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T03:34:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T03:34:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T03:34:41.170 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T03:34:41.170 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:34:41.171 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:34:41.172 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:41.173 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:41.395 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:19 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:41.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T03:34:41.534 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 139 k 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:41.535 INFO:teuthology.orchestra.run.smithi001.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout:Install 6 Packages 2024-09-14T03:34:41.536 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:41.537 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 7.0 M 2024-09-14T03:34:41.537 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 33 M 2024-09-14T03:34:41.537 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:41.753 INFO:teuthology.orchestra.run.smithi001.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.7 MB/s | 248 kB 00:00 2024-09-14T03:34:41.803 INFO:teuthology.orchestra.run.smithi001.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-09-14T03:34:41.829 INFO:teuthology.orchestra.run.smithi001.stdout:(3/6): ceph-mgr-cephadm-18.2.4-845.ge4e227b3.el 649 kB/s | 139 kB 00:00 2024-09-14T03:34:41.854 INFO:teuthology.orchestra.run.smithi001.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.1 MB/s | 58 kB 00:00 2024-09-14T03:34:41.904 INFO:teuthology.orchestra.run.smithi001.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 993 kB/s | 74 kB 00:00 2024-09-14T03:34:41.987 INFO:teuthology.orchestra.run.smithi001.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 16 MB/s | 6.0 MB 00:00 2024-09-14T03:34:41.990 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:41.990 INFO:teuthology.orchestra.run.smithi001.stdout:Total 15 MB/s | 7.0 MB 00:00 2024-09-14T03:34:42.012 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:19 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:42.063 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:42.075 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:42.075 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:42.151 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:42.152 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:42.152 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:42.152 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:42.152 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:42.152 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 139 k 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2024-09-14T03:34:42.153 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:42.154 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 7.0 M 2024-09-14T03:34:42.154 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 33 M 2024-09-14T03:34:42.154 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:42.182 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:42.182 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:42.511 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:42.682 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-09-14T03:34:42.694 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T03:34:42.707 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): ceph-mgr-cephadm-18.2.4-845.ge4e227b3.el 578 kB/s | 139 kB 00:00 2024-09-14T03:34:42.741 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T03:34:42.766 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 6.4 MB/s | 548 kB 00:00 2024-09-14T03:34:42.800 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 625 kB/s | 58 kB 00:00 2024-09-14T03:34:42.825 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.2 MB/s | 74 kB 00:00 2024-09-14T03:34:42.992 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 11 MB/s | 6.0 MB 00:00 2024-09-14T03:34:42.994 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:42.995 INFO:teuthology.orchestra.run.smithi012.stdout:Total 8.3 MB/s | 7.0 MB 00:00 2024-09-14T03:34:43.068 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:43.080 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:43.080 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:43.186 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:43.187 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:43.322 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T03:34:43.411 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T03:34:43.481 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T03:34:43.510 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:34:43.510 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:43.683 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T03:34:43.737 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T03:34:44.262 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:34:44.263 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/6 2024-09-14T03:34:44.263 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T03:34:44.263 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T03:34:44.263 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T03:34:44.263 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T03:34:44.294 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T03:34:44.391 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T03:34:44.461 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T03:34:44.497 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:34:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T03:34:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:44.560 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:44.561 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:44.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install ceph-fuse 2024-09-14T03:34:45.321 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:34:45.321 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/6 2024-09-14T03:34:45.321 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T03:34:45.321 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T03:34:45.322 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T03:34:45.322 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T03:34:45.380 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:23 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repository Size 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:45.519 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 852 k 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout:Installing dependencies: 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout:Install 2 Packages 2024-09-14T03:34:45.520 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:45.521 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 932 k 2024-09-14T03:34:45.521 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 2.7 M 2024-09-14T03:34:45.521 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:45.601 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:45.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-fuse 2024-09-14T03:34:45.973 INFO:teuthology.orchestra.run.smithi001.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 393 kB/s | 80 kB 00:00 2024-09-14T03:34:46.022 INFO:teuthology.orchestra.run.smithi001.stdout:(2/2): ceph-fuse-18.2.4-845.ge4e227b3.el9.x86_6 3.3 MB/s | 852 kB 00:00 2024-09-14T03:34:46.023 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:46.023 INFO:teuthology.orchestra.run.smithi001.stdout:Total 1.8 MB/s | 932 kB 00:00 2024-09-14T03:34:46.036 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:46.059 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:46.060 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:46.147 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:46.147 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:46.335 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:46.442 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:23 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:46.475 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T03:34:46.549 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:34:46.582 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:46.582 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:46.582 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-09-14T03:34:46.582 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:46.582 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 852 k 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout:Install 2 Packages 2024-09-14T03:34:46.583 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:46.584 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 932 k 2024-09-14T03:34:46.584 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 2.7 M 2024-09-14T03:34:46.584 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:46.712 INFO:teuthology.orchestra.run.smithi012.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 789 kB/s | 80 kB 00:00 2024-09-14T03:34:46.885 INFO:teuthology.orchestra.run.smithi012.stdout:(2/2): ceph-fuse-18.2.4-845.ge4e227b3.el9.x86_6 3.0 MB/s | 852 kB 00:00 2024-09-14T03:34:46.885 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:46.886 INFO:teuthology.orchestra.run.smithi012.stdout:Total 3.0 MB/s | 932 kB 00:00 2024-09-14T03:34:46.898 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:46.921 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:46.921 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:47.006 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:47.007 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:47.199 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:47.227 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:34:47.228 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/2 2024-09-14T03:34:47.347 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T03:34:47.437 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:34:47.528 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T03:34:47.529 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:47.529 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:47.529 INFO:teuthology.orchestra.run.smithi001.stdout: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T03:34:47.529 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:47.529 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:47.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install librados-devel 2024-09-14T03:34:48.111 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:34:48.111 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/2 2024-09-14T03:34:48.378 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:26 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:48.414 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T03:34:48.415 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:48.415 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:48.415 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T03:34:48.415 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:48.415 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:48.516 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:48.517 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:48.517 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repo Size 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout: librados-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 127 k 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:48.518 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:48.519 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:34:48.519 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:48.519 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 127 k 2024-09-14T03:34:48.519 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 456 k 2024-09-14T03:34:48.519 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:48.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados-devel 2024-09-14T03:34:48.710 INFO:teuthology.orchestra.run.smithi001.stdout:librados-devel-18.2.4-845.ge4e227b3.el9.x86_64. 659 kB/s | 127 kB 00:00 2024-09-14T03:34:48.711 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:48.711 INFO:teuthology.orchestra.run.smithi001.stdout:Total 653 kB/s | 127 kB 00:00 2024-09-14T03:34:48.711 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:48.717 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:48.717 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:48.784 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:48.784 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:48.950 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:49.052 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:49.259 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:26 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:49.397 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 127 k 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:49.398 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 127 k 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 456 k 2024-09-14T03:34:49.399 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:49.445 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:49.606 INFO:teuthology.orchestra.run.smithi012.stdout:librados-devel-18.2.4-845.ge4e227b3.el9.x86_64. 612 kB/s | 127 kB 00:00 2024-09-14T03:34:49.606 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:49.607 INFO:teuthology.orchestra.run.smithi012.stdout:Total 608 kB/s | 127 kB 00:00 2024-09-14T03:34:49.607 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:49.613 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:49.613 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:49.681 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:49.681 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:49.709 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:49.940 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:49.966 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install libcephfs2 2024-09-14T03:34:50.348 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:50.567 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:28 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:50.625 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:50.661 INFO:teuthology.orchestra.run.smithi001.stdout:Package libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:50.707 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:50.708 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:50.709 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:50.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install libcephfs-devel 2024-09-14T03:34:50.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs2 2024-09-14T03:34:51.364 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:29 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:51.419 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:28 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:51.500 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repo Size 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout: libcephfs-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 31 k 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:34:51.501 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:51.502 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 31 k 2024-09-14T03:34:51.502 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 139 k 2024-09-14T03:34:51.502 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:51.513 INFO:teuthology.orchestra.run.smithi012.stdout:Package libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:51.559 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:51.560 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:51.560 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:51.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs-devel 2024-09-14T03:34:51.664 INFO:teuthology.orchestra.run.smithi001.stdout:libcephfs-devel-18.2.4-845.ge4e227b3.el9.x86_64 191 kB/s | 31 kB 00:00 2024-09-14T03:34:51.664 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:51.665 INFO:teuthology.orchestra.run.smithi001.stdout:Total 190 kB/s | 31 kB 00:00 2024-09-14T03:34:51.665 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:51.668 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:51.668 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:51.697 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:51.697 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:51.786 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:51.889 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:52.213 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:29 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:52.265 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:52.353 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 31 k 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:52.354 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:34:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:52.355 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 31 k 2024-09-14T03:34:52.355 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 139 k 2024-09-14T03:34:52.355 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:52.508 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:52.509 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:52.509 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:52.509 INFO:teuthology.orchestra.run.smithi001.stdout: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:52.509 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:52.509 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:52.510 INFO:teuthology.orchestra.run.smithi012.stdout:libcephfs-devel-18.2.4-845.ge4e227b3.el9.x86_64 201 kB/s | 31 kB 00:00 2024-09-14T03:34:52.510 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:52.510 INFO:teuthology.orchestra.run.smithi012.stdout:Total 199 kB/s | 31 kB 00:00 2024-09-14T03:34:52.510 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:52.514 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:52.514 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:52.533 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:52.533 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:52.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install librados2 2024-09-14T03:34:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:53.272 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:31 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:53.349 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:53.367 INFO:teuthology.orchestra.run.smithi001.stdout:Package librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:53.413 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:53.414 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:53.414 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:53.473 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install librbd1 2024-09-14T03:34:53.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados2 2024-09-14T03:34:54.062 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:54.143 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:31 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:54.157 INFO:teuthology.orchestra.run.smithi001.stdout:Package librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:54.203 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:54.204 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:54.204 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:54.237 INFO:teuthology.orchestra.run.smithi012.stdout:Package librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:54.267 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install python3-rados 2024-09-14T03:34:54.282 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:54.283 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:54.283 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:54.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librbd1 2024-09-14T03:34:54.856 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:54.931 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:31 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:54.949 INFO:teuthology.orchestra.run.smithi001.stdout:Package python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:54.994 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:54.995 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:54.995 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:55.027 INFO:teuthology.orchestra.run.smithi012.stdout:Package librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:55.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install python3-rgw 2024-09-14T03:34:55.074 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:55.074 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:55.074 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:55.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rados 2024-09-14T03:34:55.662 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:33 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:55.732 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:55.757 INFO:teuthology.orchestra.run.smithi001.stdout:Package python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:55.802 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:55.803 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:55.803 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:55.825 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:55.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install python3-cephfs 2024-09-14T03:34:55.871 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:55.872 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:55.872 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:55.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rgw 2024-09-14T03:34:56.453 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:34 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:56.522 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:33 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:56.546 INFO:teuthology.orchestra.run.smithi001.stdout:Package python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:56.592 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:56.593 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:56.593 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:56.616 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:56.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install python3-rbd 2024-09-14T03:34:56.661 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:56.662 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:56.662 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:56.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-cephfs 2024-09-14T03:34:57.245 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:57.308 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:34 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:57.338 INFO:teuthology.orchestra.run.smithi001.stdout:Package python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:57.383 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:57.384 INFO:teuthology.orchestra.run.smithi001.stdout:Nothing to do. 2024-09-14T03:34:57.385 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:57.402 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:57.448 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:57.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install rbd-fuse 2024-09-14T03:34:57.449 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:57.449 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:57.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rbd 2024-09-14T03:34:58.038 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:36 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:34:58.099 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:58.179 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:34:58.179 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout: Package Architecture Version Repository Size 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 87 k 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 87 k 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 227 k 2024-09-14T03:34:58.180 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:34:58.194 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:34:58.238 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:58.239 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-09-14T03:34:58.239 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:34:58.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-fuse 2024-09-14T03:34:58.367 INFO:teuthology.orchestra.run.smithi001.stdout:rbd-fuse-18.2.4-845.ge4e227b3.el9.x86_64.rpm 466 kB/s | 87 kB 00:00 2024-09-14T03:34:58.368 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:58.368 INFO:teuthology.orchestra.run.smithi001.stdout:Total 462 kB/s | 87 kB 00:00 2024-09-14T03:34:58.368 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:34:58.377 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:34:58.377 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:34:58.452 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:34:58.452 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:34:58.617 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:34:58.834 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:58.899 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:34:59.038 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:34:59.039 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:59.039 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-09-14T03:34:59.039 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:59.039 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:34:59.039 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 87 k 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 87 k 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 227 k 2024-09-14T03:34:59.040 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:34:59.193 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:59.229 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-fuse-18.2.4-845.ge4e227b3.el9.x86_64.rpm 460 kB/s | 87 kB 00:00 2024-09-14T03:34:59.230 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:34:59.230 INFO:teuthology.orchestra.run.smithi012.stdout:Total 457 kB/s | 87 kB 00:00 2024-09-14T03:34:59.230 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:34:59.237 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:34:59.238 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:34:59.310 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:34:59.310 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:34:59.465 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:34:59.472 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:34:59.570 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:34:59.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install rbd-mirror 2024-09-14T03:34:59.958 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:00.223 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:00.223 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:00.223 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:35:00.224 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:35:00.224 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:00.224 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:35:00.271 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:38 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:35:00.411 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:35:00.412 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:00.412 INFO:teuthology.orchestra.run.smithi001.stdout: Package Arch Version Repo Size 2024-09-14T03:35:00.412 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:00.412 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:35:00.412 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-mirror x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:35:00.413 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:00.413 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:35:00.413 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:00.416 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:35:00.417 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:00.417 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 3.0 M 2024-09-14T03:35:00.418 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 12 M 2024-09-14T03:35:00.418 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:35:00.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-mirror 2024-09-14T03:35:00.762 INFO:teuthology.orchestra.run.smithi001.stdout:rbd-mirror-18.2.4-845.ge4e227b3.el9.x86_64.rpm 8.6 MB/s | 3.0 MB 00:00 2024-09-14T03:35:00.762 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:00.763 INFO:teuthology.orchestra.run.smithi001.stdout:Total 8.5 MB/s | 3.0 MB 00:00 2024-09-14T03:35:00.763 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:35:00.773 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:35:00.773 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:35:00.858 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:35:00.858 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:35:01.049 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:38 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:35:01.193 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:35:01.193 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.0 M 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 12 M 2024-09-14T03:35:01.194 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:35:01.299 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:35:01.379 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:01.405 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:01.405 INFO:teuthology.orchestra.run.smithi001.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:35:01.405 INFO:teuthology.orchestra.run.smithi001.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:35:01.405 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:35:01.406 INFO:teuthology.orchestra.run.smithi001.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:35:01.406 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:01.524 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-mirror-18.2.4-845.ge4e227b3.el9.x86_64.rpm 9.1 MB/s | 3.0 MB 00:00 2024-09-14T03:35:01.524 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:01.525 INFO:teuthology.orchestra.run.smithi012.stdout:Total 9.0 MB/s | 3.0 MB 00:00 2024-09-14T03:35:01.525 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:35:01.535 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:35:01.535 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:35:01.620 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:35:01.620 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:35:02.062 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:35:02.133 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:35:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:02.494 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:02.495 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:02.495 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:35:02.495 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:35:02.495 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:02.495 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:35:02.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo yum -y install rbd-nbd 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:03.252 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:35:03.329 INFO:teuthology.orchestra.run.smithi001.stdout:Last metadata expiration check: 0:01:41 ago on Sat 14 Sep 2024 03:33:22 AM UTC. 2024-09-14T03:35:03.468 INFO:teuthology.orchestra.run.smithi001.stdout:Dependencies resolved. 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout: Package Architecture Version Repository Size 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout:Installing: 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-nbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 172 k 2024-09-14T03:35:03.469 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction Summary 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:================================================================================ 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:Install 1 Package 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:Total download size: 172 k 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:Installed size: 494 k 2024-09-14T03:35:03.470 INFO:teuthology.orchestra.run.smithi001.stdout:Downloading Packages: 2024-09-14T03:35:03.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-nbd 2024-09-14T03:35:03.674 INFO:teuthology.orchestra.run.smithi001.stdout:rbd-nbd-18.2.4-845.ge4e227b3.el9.x86_64.rpm 846 kB/s | 172 kB 00:00 2024-09-14T03:35:03.674 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:03.675 INFO:teuthology.orchestra.run.smithi001.stdout:Total 839 kB/s | 172 kB 00:00 2024-09-14T03:35:03.675 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction check 2024-09-14T03:35:03.683 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction check succeeded. 2024-09-14T03:35:03.683 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction test 2024-09-14T03:35:03.756 INFO:teuthology.orchestra.run.smithi001.stdout:Transaction test succeeded. 2024-09-14T03:35:03.757 INFO:teuthology.orchestra.run.smithi001.stdout:Running transaction 2024-09-14T03:35:03.933 INFO:teuthology.orchestra.run.smithi001.stdout: Preparing : 1/1 2024-09-14T03:35:04.024 INFO:teuthology.orchestra.run.smithi001.stdout: Installing : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:04.087 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:41 ago on Sat 14 Sep 2024 03:33:23 AM UTC. 2024-09-14T03:35:04.226 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 172 k 2024-09-14T03:35:04.227 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 172 k 2024-09-14T03:35:04.228 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 494 k 2024-09-14T03:35:04.229 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-14T03:35:04.433 INFO:teuthology.orchestra.run.smithi001.stdout: Running scriptlet: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:04.435 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-nbd-18.2.4-845.ge4e227b3.el9.x86_64.rpm 830 kB/s | 172 kB 00:00 2024-09-14T03:35:04.436 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:04.436 INFO:teuthology.orchestra.run.smithi012.stdout:Total 823 kB/s | 172 kB 00:00 2024-09-14T03:35:04.436 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-14T03:35:04.444 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-14T03:35:04.445 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-14T03:35:04.519 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-14T03:35:04.519 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-14T03:35:04.704 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-14T03:35:04.708 INFO:teuthology.orchestra.run.smithi001.stdout: Verifying : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:04.708 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:04.708 INFO:teuthology.orchestra.run.smithi001.stdout:Installed: 2024-09-14T03:35:04.708 INFO:teuthology.orchestra.run.smithi001.stdout: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:35:04.708 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:04.709 INFO:teuthology.orchestra.run.smithi001.stdout:Complete! 2024-09-14T03:35:04.794 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:04.931 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:05.241 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:35:05.530 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-14T03:35:05.750 DEBUG:teuthology.parallel:result is None 2024-09-14T03:35:05.750 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:35:05.750 INFO:teuthology.packaging:ref: None 2024-09-14T03:35:05.750 INFO:teuthology.packaging:tag: None 2024-09-14T03:35:05.750 INFO:teuthology.packaging:branch: reef 2024-09-14T03:35:05.750 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:35:05.751 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:35:05.984 DEBUG:teuthology.orchestra.run.smithi001:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:35:06.010 INFO:teuthology.orchestra.run.smithi001.stdout:18.2.4-845.ge4e227b3.el9 2024-09-14T03:35:06.011 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-845.ge4e227b3.el9 2024-09-14T03:35:06.011 INFO:teuthology.task.install:The correct ceph version 18.2.4-845.ge4e227b3 is installed. 2024-09-14T03:35:06.013 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:35:06.013 INFO:teuthology.packaging:ref: None 2024-09-14T03:35:06.013 INFO:teuthology.packaging:tag: None 2024-09-14T03:35:06.013 INFO:teuthology.packaging:branch: reef 2024-09-14T03:35:06.013 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:35:06.013 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:35:06.251 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:35:06.273 INFO:teuthology.orchestra.run.smithi012.stdout:18.2.4-845.ge4e227b3.el9 2024-09-14T03:35:06.273 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-845.ge4e227b3.el9 2024-09-14T03:35:06.273 INFO:teuthology.task.install:The correct ceph version 18.2.4-845.ge4e227b3 is installed. 2024-09-14T03:35:06.275 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T03:35:06.275 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:35:06.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:35:06.308 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:35:06.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:35:06.340 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T03:35:06.340 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:35:06.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:35:06.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:35:06.447 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:35:06.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:35:06.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:35:06.544 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T03:35:06.544 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:35:06.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:35:06.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:35:06.640 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:35:06.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:35:06.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:35:06.733 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T03:35:06.733 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:35:06.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:35:06.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:35:06.831 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:35:06.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:35:06.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:35:06.923 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-14T03:35:07.029 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:35:07.030 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:35:07.030 INFO:tasks.cephadm:Cluster fsid is 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:07.030 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-14T03:35:07.031 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.1', 'mon.c': '[v2:172.21.15.1:3301,v1:172.21.15.1:6790]', 'mon.b': '172.21.15.12'} 2024-09-14T03:35:07.031 INFO:tasks.cephadm:First mon is mon.a on smithi001 2024-09-14T03:35:07.031 INFO:tasks.cephadm:First mgr is y 2024-09-14T03:35:07.031 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-14T03:35:07.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo hostname $(hostname -s) 2024-09-14T03:35:07.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo hostname $(hostname -s) 2024-09-14T03:35:07.088 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-09-14T03:35:07.088 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:35:07.227 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/56069/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-4907-ga2782315', 'node_name': '172.21.2.8+braggi08', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-4907.ga2782315'}, 'url': 'https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-09-12 21:34:57.306979', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-09-14T03:35:07.456 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref reef, sha1 e4e227b37858f5109138494e01e5ba1ba7b8ffe8 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-09-14T03:35:07.458 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm 2024-09-14T03:35:07.458 INFO:tasks.cephadm:Downloading cephadm from url: https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm 2024-09-14T03:35:07.458 DEBUG:teuthology.orchestra.run.smithi001:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:07.811 INFO:teuthology.orchestra.run.smithi001.stdout:-rw-r--r--. 1 ubuntu ubuntu 217223 Sep 14 03:35 /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:07.811 DEBUG:teuthology.orchestra.run.smithi012:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:08.152 INFO:teuthology.orchestra.run.smithi012.stdout:-rw-r--r--. 1 ubuntu ubuntu 217223 Sep 14 03:35 /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:08.153 DEBUG:teuthology.orchestra.run.smithi001:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:08.173 DEBUG:teuthology.orchestra.run.smithi012:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:35:08.205 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-09-14T03:35:08.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-14T03:35:08.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-14T03:35:09.764 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:35:09.908 INFO:teuthology.orchestra.run.smithi001.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:35:36.691 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph_version": "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)", 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout: "image_id": "67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1", 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout: "repo_digests": [ 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322" 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T03:35:36.692 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:35:41.212 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T03:35:41.212 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph_version": "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)", 2024-09-14T03:35:41.212 INFO:teuthology.orchestra.run.smithi012.stdout: "image_id": "67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1", 2024-09-14T03:35:41.212 INFO:teuthology.orchestra.run.smithi012.stdout: "repo_digests": [ 2024-09-14T03:35:41.213 INFO:teuthology.orchestra.run.smithi012.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322" 2024-09-14T03:35:41.213 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T03:35:41.213 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T03:35:41.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo mkdir -p /etc/ceph 2024-09-14T03:35:41.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph 2024-09-14T03:35:41.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod 777 /etc/ceph 2024-09-14T03:35:41.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 777 /etc/ceph 2024-09-14T03:35:41.363 INFO:tasks.cephadm:Writing seed config... 2024-09-14T03:35:41.364 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-14T03:35:41.364 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-14T03:35:41.364 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-14T03:35:41.364 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-14T03:35:41.365 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-14T03:35:41.366 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:35:41.366 DEBUG:teuthology.orchestra.run.smithi001:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-14T03:35:41.395 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5626d550-724a-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-14T03:35:41.395 DEBUG:teuthology.orchestra.run.smithi001:mon.a> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service 2024-09-14T03:35:41.438 DEBUG:teuthology.orchestra.run.smithi001:mgr.y> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service 2024-09-14T03:35:41.481 INFO:tasks.cephadm:Bootstrapping... 2024-09-14T03:35:41.481 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid 5626d550-724a-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.1 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:35:41.643 INFO:teuthology.orchestra.run.smithi001.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:41.644 INFO:teuthology.orchestra.run.smithi001.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', '5626d550-724a-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.1', '--skip-admin-label'] 2024-09-14T03:35:41.673 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:35:41.673 INFO:teuthology.orchestra.run.smithi001.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-14T03:35:41.674 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying podman|docker is present... 2024-09-14T03:35:41.701 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:35:41.701 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying lvm2 is present... 2024-09-14T03:35:41.702 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying time synchronization is in place... 2024-09-14T03:35:41.707 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:35:41.708 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:35:41.713 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:35:41.713 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:35:41.720 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout enabled 2024-09-14T03:35:41.728 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout active 2024-09-14T03:35:41.728 INFO:teuthology.orchestra.run.smithi001.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:35:41.728 INFO:teuthology.orchestra.run.smithi001.stdout:Repeating the final host check... 2024-09-14T03:35:41.755 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:35:41.755 INFO:teuthology.orchestra.run.smithi001.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-14T03:35:41.755 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl is present 2024-09-14T03:35:41.755 INFO:teuthology.orchestra.run.smithi001.stdout:lvcreate is present 2024-09-14T03:35:41.762 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:35:41.763 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:35:41.770 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:35:41.770 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:35:41.777 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout enabled 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout active 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:Host looks OK 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:Cluster fsid: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:Acquiring lock 140015269482064 on /run/cephadm/5626d550-724a-11ef-bceb-c7b262605968.lock 2024-09-14T03:35:41.785 INFO:teuthology.orchestra.run.smithi001.stdout:Lock 140015269482064 acquired on /run/cephadm/5626d550-724a-11ef-bceb-c7b262605968.lock 2024-09-14T03:35:41.786 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying IP 172.21.15.1 port 3300 ... 2024-09-14T03:35:41.786 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying IP 172.21.15.1 port 6789 ... 2024-09-14T03:35:41.786 INFO:teuthology.orchestra.run.smithi001.stdout:Base mon IP(s) is [172.21.15.1:3300, 172.21.15.1:6789], mon addrv is [v2:172.21.15.1:3300,v1:172.21.15.1:6789] 2024-09-14T03:35:41.789 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.1 metric 100 2024-09-14T03:35:41.789 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.1 metric 100 2024-09-14T03:35:41.792 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-14T03:35:41.792 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-14T03:35:41.792 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1795sec hoplimit 64 pref medium 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:15e8/64 scope link 2024-09-14T03:35:41.795 INFO:teuthology.orchestra.run.smithi001.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:35:41.796 INFO:teuthology.orchestra.run.smithi001.stdout:Mon IP `172.21.15.1` is in CIDR network `172.21.0.0/20` 2024-09-14T03:35:41.796 INFO:teuthology.orchestra.run.smithi001.stdout:Mon IP `172.21.15.1` is in CIDR network `172.21.0.0/20` 2024-09-14T03:35:41.797 INFO:teuthology.orchestra.run.smithi001.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-14T03:35:41.797 INFO:teuthology.orchestra.run.smithi001.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-14T03:35:41.797 INFO:teuthology.orchestra.run.smithi001.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stdout 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Getting image source signatures 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Copying blob sha256:71878612010b3dfa9e3e456d432a49e20ac26634831a8a6522f4d782181c5840 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Copying config sha256:67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 2024-09-14T03:35:42.393 INFO:teuthology.orchestra.run.smithi001.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-14T03:35:43.381 INFO:teuthology.orchestra.run.smithi001.stdout:ceph: stdout ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable) 2024-09-14T03:35:43.382 INFO:teuthology.orchestra.run.smithi001.stdout:Ceph version: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable) 2024-09-14T03:35:43.382 INFO:teuthology.orchestra.run.smithi001.stdout:Extracting ceph user uid/gid from container image... 2024-09-14T03:35:44.396 INFO:teuthology.orchestra.run.smithi001.stdout:stat: stdout 167 167 2024-09-14T03:35:44.396 INFO:teuthology.orchestra.run.smithi001.stdout:Creating initial keys... 2024-09-14T03:35:45.318 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph-authtool: stdout AQCQBOVmfZn6LBAA0+eM8JiAn5P+80Myxbwc6g== 2024-09-14T03:35:46.266 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph-authtool: stdout AQCRBOVm1NvzKRAAuXXWeVNdjn2tpcc80yGPjQ== 2024-09-14T03:35:47.263 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph-authtool: stdout AQCSBOVmPRJdIxAAF9QHfw/XQJeHuQlqL4Rb9Q== 2024-09-14T03:35:47.264 INFO:teuthology.orchestra.run.smithi001.stdout:Creating initial monmap... 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:monmaptool for a [v2:172.21.15.1:3300,v1:172.21.15.1:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:setting min_mon_release = pacific 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: set fsid to 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:35:48.145 INFO:teuthology.orchestra.run.smithi001.stdout:Creating mon... 2024-09-14T03:35:49.058 INFO:teuthology.orchestra.run.smithi001.stdout:create mon.a on 2024-09-14T03:35:49.276 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-14T03:35:49.424 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-14T03:35:49.616 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5626d550-724a-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968.target. 2024-09-14T03:35:49.616 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-5626d550-724a-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968.target. 2024-09-14T03:35:49.842 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a 2024-09-14T03:35:49.842 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Failed to reset failed state of unit ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-14T03:35:50.015 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968.target.wants/ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968@.service. 2024-09-14T03:35:50.498 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:50 smithi001 podman[28849]: 2024-09-14 03:35:50.456472449 +0000 UTC m=+0.317173750 container start 231fcd8f6b1a8fb269740ff002d08f5d026dba1c84f7d477ddc961eecf20f434 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T03:35:50.498 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:50 smithi001 ceph-mon[28887]: mkfs 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:50.558 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:35:50.558 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout disabled 2024-09-14T03:35:50.565 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:35:50.565 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:35:50.565 INFO:teuthology.orchestra.run.smithi001.stdout:firewalld.service is not enabled 2024-09-14T03:35:50.565 INFO:teuthology.orchestra.run.smithi001.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:35:50.565 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mon to start... 2024-09-14T03:35:50.566 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mon... 2024-09-14T03:35:50.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:50 smithi001 ceph-mon[28887]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:35:50.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:50 smithi001 bash[28849]: 231fcd8f6b1a8fb269740ff002d08f5d026dba1c84f7d477ddc961eecf20f434 2024-09-14T03:35:50.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:50 smithi001 systemd[1]: Started Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: monmap e1: 1 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: fsmap 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: mgrmap e1: no daemons active 2024-09-14T03:35:51.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:51 smithi001 ceph-mon[28887]: from='client.? 172.21.15.1:0/1426209638' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-14T03:35:51.628 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout cluster: 2024-09-14T03:35:51.628 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout id: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:51.628 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout services: 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.544201s) 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout data: 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout pgs: 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:mon is available 2024-09-14T03:35:51.629 INFO:teuthology.orchestra.run.smithi001.stdout:Assimilating anything we can from ceph.conf... 2024-09-14T03:35:52.676 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:52.676 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:35:52.676 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout fsid = 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.1:3300,v1:172.21.15.1:6789] 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:35:52.677 INFO:teuthology.orchestra.run.smithi001.stdout:Generating new minimal ceph.conf... 2024-09-14T03:35:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:52 smithi001 ceph-mon[28887]: from='client.? 172.21.15.1:0/1313597442' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:35:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:52 smithi001 ceph-mon[28887]: from='client.? 172.21.15.1:0/1313597442' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-14T03:35:53.749 INFO:teuthology.orchestra.run.smithi001.stdout:Restarting the monitor... 2024-09-14T03:35:53.773 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:53 smithi001 ceph-mon[28887]: from='client.? 172.21.15.1:0/280476195' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:35:54.043 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:53 smithi001 systemd[1]: Stopping Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:35:54.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[28864]: 2024-09-14T03:35:54.041+0000 7f3bbe4fe640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:35:54.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[28864]: 2024-09-14T03:35:54.041+0000 7f3bbe4fe640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-14T03:35:54.426 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 podman[29206]: 2024-09-14 03:35:54.185354075 +0000 UTC m=+0.241263749 container died 231fcd8f6b1a8fb269740ff002d08f5d026dba1c84f7d477ddc961eecf20f434 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T03:35:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 podman[29206]: 2024-09-14 03:35:54.522775875 +0000 UTC m=+0.578685540 container cleanup 231fcd8f6b1a8fb269740ff002d08f5d026dba1c84f7d477ddc961eecf20f434 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T03:35:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 bash[29206]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:35:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 podman[29223]: 2024-09-14 03:35:54.665240232 +0000 UTC m=+0.475497817 container remove 231fcd8f6b1a8fb269740ff002d08f5d026dba1c84f7d477ddc961eecf20f434 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:35:55.075 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:35:55.075 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 systemd[1]: Stopped Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:35:55.075 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:54 smithi001 systemd[1]: Starting Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:35:55.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 podman[29323]: 2024-09-14 03:35:55.07463304 +0000 UTC m=+0.096608610 container create 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T03:35:55.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 podman[29323]: 2024-09-14 03:35:54.994573354 +0000 UTC m=+0.016548938 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:35:55.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 podman[29323]: 2024-09-14 03:35:55.238906775 +0000 UTC m=+0.260882349 container init 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-14T03:35:55.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 podman[29323]: 2024-09-14 03:35:55.243940657 +0000 UTC m=+0.265916228 container start 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0) 2024-09-14T03:35:55.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable), process ceph-mon, pid 7 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: pidfile_write: ignore empty --pid-file 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: load: jerasure load: lrc 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Git sha 0 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Compile date 2024-09-13 16:45:57 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: DB SUMMARY 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: DB Session ID: BNX716OF3AHIBX0GML2G 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:35:55.333 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84514 ; 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.env: 0x564176365bc0 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.info_log: 0x564178b70be0 2024-09-14T03:35:55.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.statistics: (nil) 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.use_fsync: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:35:55.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.db_log_dir: 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.wal_dir: 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.write_buffer_manager: 0x564178b7e320 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:35:55.336 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:35:55.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.unordered_write: 0 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.row_cache: None 2024-09-14T03:35:55.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.wal_filter: None 2024-09-14T03:35:55.339 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:35:55.339 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:35:55.339 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:35:55.339 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.wal_compression: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:35:55.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:35:55.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_open_files: -1 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:35:55.342 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Compression algorithms supported: 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kZSTD supported: 0 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.merge_operator: 2024-09-14T03:35:55.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_filter: None 2024-09-14T03:35:55.344 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:35:55.344 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:35:55.344 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:35:55.344 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:35:55.344 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564178b707e0) 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_type: 0 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: data_block_index_type: 0 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_shortening: 1 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: checksum: 4 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: no_block_cache: 0 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache: 0x564178b65350 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:35:55.345 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_options: 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: capacity : 536870912 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: num_shard_bits : 4 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: strict_capacity_limit : 0 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_compressed: (nil) 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: persistent_cache: (nil) 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_size: 4096 2024-09-14T03:35:55.346 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_size_deviation: 10 2024-09-14T03:35:55.347 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_restart_interval: 16 2024-09-14T03:35:55.347 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_block_restart_interval: 1 2024-09-14T03:35:55.347 INFO:journalctl@ceph.mon.a.smithi001.stdout: metadata_block_size: 4096 2024-09-14T03:35:55.347 INFO:journalctl@ceph.mon.a.smithi001.stdout: partition_filters: 0 2024-09-14T03:35:55.347 INFO:journalctl@ceph.mon.a.smithi001.stdout: use_delta_encoding: 1 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: filter_policy: bloomfilter 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: whole_key_filtering: 1 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: verify_compression: 0 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: format_version: 5 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: enable_index_compression: 1 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_align: 0 2024-09-14T03:35:55.348 INFO:journalctl@ceph.mon.a.smithi001.stdout: max_auto_readahead_size: 262144 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout: prepopulate_block_cache: 0 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:35:55.349 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression: NoCompression 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.num_levels: 7 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:35:55.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:35:55.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:35:55.352 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:35:55.353 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:35:55.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:35:55.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:35:55.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:35:55.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:35:55.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:35:55.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:35:55.358 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.ttl: 2592000 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:35:55.359 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 384c41e5-9a68-4872-8924-4edf54067175 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284955270409, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-09-14T03:35:55.360 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284955271437, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81064, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79214, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73415, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726284955, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "384c41e5-9a68-4872-8924-4edf54067175", "db_session_id": "BNX716OF3AHIBX0GML2G", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726284955271533, "job": 1, "event": "recovery_finished"} 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564178b90e00 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: DB pointer 0x564178c84000 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** DB Stats ** 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:35:55.361 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: L0 2/0 81.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 129.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Sum 2/0 81.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 129.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 129.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.362 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 129.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:35:55.363 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative compaction: 0.00 GB write, 23.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval compaction: 0.00 GB write, 23.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: Block cache BinnedLRUCache@0x564178b65350#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: starting mon.a rank 0 at public addrs [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] at bind addrs [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???) e1 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).mds e1 new map 2024-09-14T03:35:55.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).mds e1 print_map 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: e1 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: legacy client fscid: -1 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout: No filesystems configured 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: monmap e1: 1 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: fsmap 2024-09-14T03:35:55.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:35:55.366 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 ceph-mon[29360]: mgrmap e1: no daemons active 2024-09-14T03:35:55.366 INFO:teuthology.orchestra.run.smithi001.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-14T03:35:55.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 bash[29323]: 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 2024-09-14T03:35:55.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:55 smithi001 systemd[1]: Started Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:35:56.336 INFO:teuthology.orchestra.run.smithi001.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-14T03:35:56.350 INFO:teuthology.orchestra.run.smithi001.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:35:56.350 INFO:teuthology.orchestra.run.smithi001.stdout:Creating mgr... 2024-09-14T03:35:56.350 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-14T03:35:56.351 INFO:teuthology.orchestra.run.smithi001.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-14T03:35:56.585 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y 2024-09-14T03:35:56.586 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Failed to reset failed state of unit ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-14T03:35:56.739 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968.target.wants/ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-5626d550-724a-11ef-bceb-c7b262605968@.service. 2024-09-14T03:35:57.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:56 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2422143439' entity='client.admin' 2024-09-14T03:35:57.117 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:56 smithi001 podman[29597]: 2024-09-14 03:35:56.878918226 +0000 UTC m=+0.016194216 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:35:57.277 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:35:57.278 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout disabled 2024-09-14T03:35:57.285 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:35:57.285 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:35:57.285 INFO:teuthology.orchestra.run.smithi001.stdout:firewalld.service is not enabled 2024-09-14T03:35:57.285 INFO:teuthology.orchestra.run.smithi001.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:35:57.294 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:35:57.294 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout disabled 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:firewalld.service is not enabled 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mgr to start... 2024-09-14T03:35:57.302 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mgr... 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 podman[29597]: 2024-09-14 03:35:57.203967096 +0000 UTC m=+0.341243073 container init 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 podman[29597]: 2024-09-14 03:35:57.208897533 +0000 UTC m=+0.346173512 container start 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, RELEASE=reef-e4e227b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 bash[29597]: 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 systemd[1]: Started Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.336+0000 7fd48f29a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:35:57.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.395+0000 7fd48f29a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:35:57.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.556+0000 7fd48f29a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:35:57.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.629+0000 7fd48f29a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:35:57.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.783+0000 7fd48f29a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:35:58.168 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2402853549' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:35:58.169 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:57.934+0000 7fd48f29a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:35:58.419 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:58.260+0000 7fd48f29a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:35:58.419 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:58.322+0000 7fd48f29a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:35:58.422 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:35:58.427 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:35:58.427 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "fsid": "5626d550-724a-11ef-bceb-c7b262605968", 2024-09-14T03:35:58.427 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:35:58.427 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:35:58.427 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:35:58.428 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:35:58.429 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:35:58.430 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:35:50.495196+0000", 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:35:58.431 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:35:58.432 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:35:58.432 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:35:58.432 INFO:teuthology.orchestra.run.smithi001.stdout:mgr not available, waiting (1/15)... 2024-09-14T03:35:58.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:58.515+0000 7fd48f29a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:35:59.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:59 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:59.424+0000 7fd48f29a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:35:59.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:35:59 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:35:59.492+0000 7fd48f29a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:36:00.515 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.244+0000 7fd48f29a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:36:00.515 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.364+0000 7fd48f29a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:36:00.515 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.514+0000 7fd48f29a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:36:00.803 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.578+0000 7fd48f29a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:36:00.803 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.731+0000 7fd48f29a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:36:00.803 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.801+0000 7fd48f29a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:36:01.094 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:00.863+0000 7fd48f29a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:36:01.094 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.025+0000 7fd48f29a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:36:01.094 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.093+0000 7fd48f29a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:36:01.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.162+0000 7fd48f29a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:36:01.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.248+0000 7fd48f29a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:36:01.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.310+0000 7fd48f29a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:36:01.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:01.371+0000 7fd48f29a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:36:02.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:02.185+0000 7fd48f29a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:36:02.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: Activating manager daemon y 2024-09-14T03:36:02.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: mgrmap e2: y(active, starting, since 0.00295461s) 2024-09-14T03:36:02.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: Manager daemon y is now available 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' 2024-09-14T03:36:02.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:02 smithi001 ceph-mon[29360]: from='mgr.14100 172.21.15.1:0/3686336710' entity='mgr.y' 2024-09-14T03:36:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:04 smithi001 ceph-mon[29360]: mgrmap e3: y(active, since 1.00717s) 2024-09-14T03:36:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:04 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3503189840' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:36:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:05 smithi001 ceph-mon[29360]: mgrmap e4: y(active, since 2s) 2024-09-14T03:36:09.646 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "fsid": "5626d550-724a-11ef-bceb-c7b262605968", 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:36:09.647 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:36:09.648 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:36:09.651 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:36:09.652 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.653 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:35:50.495196+0000", 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:36:09.654 INFO:teuthology.orchestra.run.smithi001.stdout:mgr is available 2024-09-14T03:36:10.658 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:10 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2946424542' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:36:10.896 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:36:10.896 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout fsid = 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:36:10.897 INFO:teuthology.orchestra.run.smithi001.stdout:Enabling cephadm module... 2024-09-14T03:36:11.890 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2542702517' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-14T03:36:11.890 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setuser ceph since I am not root 2024-09-14T03:36:11.890 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setgroup ceph since I am not root 2024-09-14T03:36:11.890 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:11.718+0000 7fe93d0b4040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:36:11.890 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:11.776+0000 7fe93d0b4040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:36:12.164 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:11.932+0000 7fe93d0b4040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:36:12.164 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.000+0000 7fe93d0b4040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:36:12.164 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.162+0000 7fe93d0b4040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:36:12.449 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.309+0000 7fe93d0b4040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:36:12.703 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.635+0000 7fe93d0b4040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:36:12.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2542702517' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-14T03:36:12.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-mon[29360]: mgrmap e5: y(active, since 9s) 2024-09-14T03:36:12.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.701+0000 7fe93d0b4040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:36:12.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:12.892+0000 7fe93d0b4040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for the mgr to restart... 2024-09-14T03:36:13.667 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mgr epoch 5... 2024-09-14T03:36:13.887 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:13 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2553542860' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:36:13.887 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:13.813+0000 7fe93d0b4040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:36:13.887 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:13.885+0000 7fe93d0b4040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:36:14.924 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:14.653+0000 7fe93d0b4040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:36:14.924 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:14.773+0000 7fe93d0b4040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:36:15.181 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:14.922+0000 7fe93d0b4040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:36:15.181 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:14.986+0000 7fe93d0b4040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:36:15.181 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.139+0000 7fe93d0b4040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:36:15.438 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.212+0000 7fe93d0b4040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:36:15.438 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.274+0000 7fe93d0b4040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:36:15.438 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.436+0000 7fe93d0b4040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:36:15.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.504+0000 7fe93d0b4040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:36:15.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.573+0000 7fe93d0b4040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:36:15.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.659+0000 7fe93d0b4040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:36:16.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.720+0000 7fe93d0b4040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:36:16.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:15.781+0000 7fe93d0b4040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:16.591+0000 7fe93d0b4040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: Active manager daemon y restarted 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: Activating manager daemon y 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: osdmap e2: 0 total, 0 up, 0 in 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: mgrmap e6: y(active, starting, since 0.00320309s) 2024-09-14T03:36:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: Manager daemon y is now available 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: Found migration_current of "None". Setting to last migration. 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:16.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:36:16.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:16 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:36:18.201 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:36:18.201 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-14T03:36:18.202 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:36:18.202 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:36:18.202 INFO:teuthology.orchestra.run.smithi001.stdout:mgr epoch 5 is available 2024-09-14T03:36:18.202 INFO:teuthology.orchestra.run.smithi001.stdout:Setting orchestrator backend to cephadm... 2024-09-14T03:36:18.827 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:18 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:18.827 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:18 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:18.827 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:18 smithi001 ceph-mon[29360]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:36:18.827 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:18 smithi001 ceph-mon[29360]: mgrmap e7: y(active, since 1.00519s) 2024-09-14T03:36:18.827 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:18 smithi001 ceph-mon[29360]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:19.169+0000 7fe8a4d94640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:36:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Traceback (most recent call last): 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._connections.run(self.expiration_interval) 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:36:19.414 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._run(expiration_interval) 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s = self.context.wrap_socket( 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: return self.sslsocket_class._create( 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:36:19.415 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self.do_handshake() 2024-09-14T03:36:19.416 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:36:19.416 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._sslobj.do_handshake() 2024-09-14T03:36:19.416 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:36:19.416 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:18] ENGINE Bus STARTING 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:19] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:19] ENGINE Bus STARTED 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:36:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:36:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:36:20.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:36:20.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:36:20.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:36:20.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:20 smithi001 ceph-mon[29360]: mgrmap e8: y(active, since 2s) 2024-09-14T03:36:21.017 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-14T03:36:21.017 INFO:teuthology.orchestra.run.smithi001.stdout:Generating ssh key... 2024-09-14T03:36:21.374 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-mon[29360]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Generating public/private rsa key pair. 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Your identification has been saved in /tmp/tmp4kuz902e/key 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Your public key has been saved in /tmp/tmp4kuz902e/key.pub 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: The key fingerprint is: 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: SHA256:lH03YG+5Bn7oVbMQq5ZNqcpBMuCyiR4ImNiHge/ivkM ceph-5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: The key's randomart image is: 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: +---[RSA 3072]----+ 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: | .. . o. | 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |+o o. . o . o+. | 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |=.o... oo.. o=*..| 2024-09-14T03:36:22.053 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |o o.+ .+ o*=.+o| 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |.+ o S. =o.=. | 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |oEo . +. + | 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |oo o . | 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: | o | 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: |.oo | 2024-09-14T03:36:22.054 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: +----[SHA256]-----+ 2024-09-14T03:36:22.303 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:22 smithi001 ceph-mon[29360]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:22.303 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:22 smithi001 ceph-mon[29360]: Generating ssh key... 2024-09-14T03:36:22.303 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:22 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:22.303 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:22 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:23.528 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOFQYcREEDvWG7GoLrTdkkx1RknDUvKqSb7EmGT+hJxyalMYinZLy0UpFlye2ULLWRc5/Rsk1NcXOuY+opI+x8KGs6R2acF/wHWnl0u79kUJEaDtmSZ5W7DK2NmWkjTgqTDxX47R4om4ziIlDUEjMmGCnHM2ISjEPVLB+2dip+nGVDhBApDq9dZ5rK0TGxqY2m6wdkxqXlMXl6qHOEa+V/oTle3pFnYPf1e1zrl5d+HxZ83plPNhtYFSe1HcU+Azqpr/HOILykKBtpVFKN4zy7cj8Q8yUJGRdA74FzDSW4HE4G2JF3cc4QjQ9jfnTFRV2NoP8MOmMmT00d6/xDBph7be9LMF1JHkvV3aUhxjOvE5ShtlEv5PwxwGHqzfdiUiQjDLFCPf/3btToVSJ4ppu4q1CJDrf264vUfnLlUbMlA4uNJPeIEA0ERs94mXaiiGDBMYkDFNsZlt3XIkkoJQ653y7wkBryC0rULIcZ6TQDxTG7ozVg8zkrLMdZtoE6Ac= ceph-5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:36:23.529 INFO:teuthology.orchestra.run.smithi001.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:36:23.529 INFO:teuthology.orchestra.run.smithi001.stdout:Adding key to root@localhost authorized_keys... 2024-09-14T03:36:23.539 INFO:teuthology.orchestra.run.smithi001.stdout:Adding host smithi001... 2024-09-14T03:36:23.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:23 smithi001 ceph-mon[29360]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:24 smithi001 ceph-mon[29360]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi001", "addr": "172.21.15.1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:25 smithi001 ceph-mon[29360]: Deploying cephadm binary to smithi001 2024-09-14T03:36:28.178 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout Added host 'smithi001' with addr '172.21.15.1' 2024-09-14T03:36:28.178 INFO:teuthology.orchestra.run.smithi001.stdout:Deploying unmanaged mon service... 2024-09-14T03:36:28.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:28 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:28.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:28 smithi001 ceph-mon[29360]: Added host smithi001 2024-09-14T03:36:28.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:28 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:29.655 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-14T03:36:29.655 INFO:teuthology.orchestra.run.smithi001.stdout:Deploying unmanaged mgr service... 2024-09-14T03:36:30.310 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:30 smithi001 ceph-mon[29360]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:30.311 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:30 smithi001 ceph-mon[29360]: Saving service mon spec with placement count:5 2024-09-14T03:36:30.311 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:30 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:31.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:31 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:31.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:31 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:31.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:31 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:31.465 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-14T03:36:32.295 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:32 smithi001 ceph-mon[29360]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:32.296 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:32 smithi001 ceph-mon[29360]: Saving service mgr spec with placement count:2 2024-09-14T03:36:33.199 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:33 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:33.200 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:33 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2035754219' entity='client.admin' 2024-09-14T03:36:34.079 INFO:teuthology.orchestra.run.smithi001.stdout:Enabling the dashboard module... 2024-09-14T03:36:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:34 smithi001 ceph-mon[29360]: from='mgr.14116 172.21.15.1:0/591974531' entity='mgr.y' 2024-09-14T03:36:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:34 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1315376155' entity='client.admin' 2024-09-14T03:36:35.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3138911468' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-14T03:36:35.670 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setuser ceph since I am not root 2024-09-14T03:36:35.670 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setgroup ceph since I am not root 2024-09-14T03:36:35.670 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:35.623+0000 7f87889d4040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:36:35.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:35.683+0000 7f87889d4040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:36:35.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:35.839+0000 7f87889d4040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:36:35.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:35.909+0000 7f87889d4040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:36:36.212 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:36.063+0000 7f87889d4040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:36:36.213 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:36.210+0000 7f87889d4040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:36:36.803 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3138911468' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-14T03:36:36.804 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-mon[29360]: mgrmap e9: y(active, since 18s) 2024-09-14T03:36:36.804 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:36.541+0000 7f87889d4040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:36:36.804 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:36.607+0000 7f87889d4040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:36:36.804 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:36.801+0000 7f87889d4040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:36:37.449 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:36:37.449 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for the mgr to restart... 2024-09-14T03:36:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:Waiting for mgr epoch 9... 2024-09-14T03:36:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:37 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3073994490' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:36:38.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:37.726+0000 7f87889d4040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:36:38.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:37.796+0000 7f87889d4040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:36:38.806 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:38.539+0000 7f87889d4040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:36:38.806 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:38.658+0000 7f87889d4040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:36:39.091 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:38.804+0000 7f87889d4040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:36:39.091 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:38.867+0000 7f87889d4040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:36:39.091 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.020+0000 7f87889d4040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:36:39.091 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.090+0000 7f87889d4040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:36:39.383 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.152+0000 7f87889d4040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:36:39.384 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.315+0000 7f87889d4040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:36:39.384 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.382+0000 7f87889d4040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:36:39.659 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.450+0000 7f87889d4040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:36:39.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.536+0000 7f87889d4040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:36:39.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.597+0000 7f87889d4040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:36:39.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:39.658+0000 7f87889d4040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:40.468+0000 7f87889d4040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: Active manager daemon y restarted 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: Activating manager daemon y 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: osdmap e3: 0 total, 0 up, 0 in 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: mgrmap e10: y(active, starting, since 0.00309431s) 2024-09-14T03:36:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: Manager daemon y is now available 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:36:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:mgr epoch 9 is available 2024-09-14T03:36:42.401 INFO:teuthology.orchestra.run.smithi001.stdout:Generating a dashboard self-signed certificate... 2024-09-14T03:36:42.658 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-mon[29360]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-mon[29360]: mgrmap e11: y(active, since 1.00566s) 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-mon[29360]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:42.429+0000 7f86eee1e640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:36:42] ENGINE Error in HTTPServer.serve 2024-09-14T03:36:42.659 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Traceback (most recent call last): 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._connections.run(self.expiration_interval) 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._run(expiration_interval) 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s = self.context.wrap_socket( 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: return self.sslsocket_class._create( 2024-09-14T03:36:42.660 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:36:42.661 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self.do_handshake() 2024-09-14T03:36:42.661 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:36:42.661 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._sslobj.do_handshake() 2024-09-14T03:36:42.661 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:36:42.661 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:36:43.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:42] ENGINE Bus STARTING 2024-09-14T03:36:43.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:42] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:36:43.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:42] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:36:43.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:42] ENGINE Bus STARTED 2024-09-14T03:36:43.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: [14/Sep/2024:03:36:42] ENGINE Error in HTTPServer.serve 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:36:43.867 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:36:43.868 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:36:43.869 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:36:43.869 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:43.869 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:43.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:43 smithi001 ceph-mon[29360]: mgrmap e12: y(active, since 2s) 2024-09-14T03:36:44.110 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-14T03:36:44.111 INFO:teuthology.orchestra.run.smithi001.stdout:Creating initial admin user... 2024-09-14T03:36:44.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:44 smithi001 ceph-mon[29360]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:45.995 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$HW8mDEi82toQJ5A9OlF/EeV5ven/0nQm9wwSaVF9Ser2zq3FzdM02", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726285005, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-14T03:36:45.995 INFO:teuthology.orchestra.run.smithi001.stdout:Fetching dashboard port number... 2024-09-14T03:36:46.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:46 smithi001 ceph-mon[29360]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:46.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:46 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:47.067 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stdout 8443 2024-09-14T03:36:47.078 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:36:47.078 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout disabled 2024-09-14T03:36:47.085 INFO:teuthology.orchestra.run.smithi001.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:36:47.086 INFO:teuthology.orchestra.run.smithi001.stdout:systemctl: stdout inactive 2024-09-14T03:36:47.086 INFO:teuthology.orchestra.run.smithi001.stdout:firewalld.service is not enabled 2024-09-14T03:36:47.086 INFO:teuthology.orchestra.run.smithi001.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout:Ceph Dashboard is now available at: 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: URL: https://smithi001.front.sepia.ceph.com:8443/ 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: User: admin 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: Password: 6e30r0u3n0 2024-09-14T03:36:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:47.089 INFO:teuthology.orchestra.run.smithi001.stdout:Saving cluster configuration to /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config directory 2024-09-14T03:36:47.089 INFO:teuthology.orchestra.run.smithi001.stdout:Enabling autotune for osd_memory_target 2024-09-14T03:36:47.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:47 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1058565546' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-14T03:36:48.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:48 smithi001 ceph-mon[29360]: mgrmap e13: y(active, since 6s) 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout:Or, if you are only running a single cluster on this host: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.649 INFO:teuthology.orchestra.run.smithi001.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: ceph telemetry on 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout:For more information see: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:36:49.650 INFO:teuthology.orchestra.run.smithi001.stdout:Bootstrap complete. 2024-09-14T03:36:49.689 INFO:tasks.cephadm:Fetching config... 2024-09-14T03:36:49.689 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:36:49.689 DEBUG:teuthology.orchestra.run.smithi001:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-14T03:36:49.708 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-14T03:36:49.708 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:36:49.708 DEBUG:teuthology.orchestra.run.smithi001:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-14T03:36:49.766 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-14T03:36:49.766 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:36:49.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd if=/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-14T03:36:49.833 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-14T03:36:49.833 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:36:49.833 DEBUG:teuthology.orchestra.run.smithi001:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-14T03:36:49.889 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-14T03:36:49.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOFQYcREEDvWG7GoLrTdkkx1RknDUvKqSb7EmGT+hJxyalMYinZLy0UpFlye2ULLWRc5/Rsk1NcXOuY+opI+x8KGs6R2acF/wHWnl0u79kUJEaDtmSZ5W7DK2NmWkjTgqTDxX47R4om4ziIlDUEjMmGCnHM2ISjEPVLB+2dip+nGVDhBApDq9dZ5rK0TGxqY2m6wdkxqXlMXl6qHOEa+V/oTle3pFnYPf1e1zrl5d+HxZ83plPNhtYFSe1HcU+Azqpr/HOILykKBtpVFKN4zy7cj8Q8yUJGRdA74FzDSW4HE4G2JF3cc4QjQ9jfnTFRV2NoP8MOmMmT00d6/xDBph7be9LMF1JHkvV3aUhxjOvE5ShtlEv5PwxwGHqzfdiUiQjDLFCPf/3btToVSJ4ppu4q1CJDrf264vUfnLlUbMlA4uNJPeIEA0ERs94mXaiiGDBMYkDFNsZlt3XIkkoJQ653y7wkBryC0rULIcZ6TQDxTG7ozVg8zkrLMdZtoE6Ac= ceph-5626d550-724a-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:36:49.965 INFO:teuthology.orchestra.run.smithi001.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOFQYcREEDvWG7GoLrTdkkx1RknDUvKqSb7EmGT+hJxyalMYinZLy0UpFlye2ULLWRc5/Rsk1NcXOuY+opI+x8KGs6R2acF/wHWnl0u79kUJEaDtmSZ5W7DK2NmWkjTgqTDxX47R4om4ziIlDUEjMmGCnHM2ISjEPVLB+2dip+nGVDhBApDq9dZ5rK0TGxqY2m6wdkxqXlMXl6qHOEa+V/oTle3pFnYPf1e1zrl5d+HxZ83plPNhtYFSe1HcU+Azqpr/HOILykKBtpVFKN4zy7cj8Q8yUJGRdA74FzDSW4HE4G2JF3cc4QjQ9jfnTFRV2NoP8MOmMmT00d6/xDBph7be9LMF1JHkvV3aUhxjOvE5ShtlEv5PwxwGHqzfdiUiQjDLFCPf/3btToVSJ4ppu4q1CJDrf264vUfnLlUbMlA4uNJPeIEA0ERs94mXaiiGDBMYkDFNsZlt3XIkkoJQ653y7wkBryC0rULIcZ6TQDxTG7ozVg8zkrLMdZtoE6Ac= ceph-5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:36:49.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOFQYcREEDvWG7GoLrTdkkx1RknDUvKqSb7EmGT+hJxyalMYinZLy0UpFlye2ULLWRc5/Rsk1NcXOuY+opI+x8KGs6R2acF/wHWnl0u79kUJEaDtmSZ5W7DK2NmWkjTgqTDxX47R4om4ziIlDUEjMmGCnHM2ISjEPVLB+2dip+nGVDhBApDq9dZ5rK0TGxqY2m6wdkxqXlMXl6qHOEa+V/oTle3pFnYPf1e1zrl5d+HxZ83plPNhtYFSe1HcU+Azqpr/HOILykKBtpVFKN4zy7cj8Q8yUJGRdA74FzDSW4HE4G2JF3cc4QjQ9jfnTFRV2NoP8MOmMmT00d6/xDBph7be9LMF1JHkvV3aUhxjOvE5ShtlEv5PwxwGHqzfdiUiQjDLFCPf/3btToVSJ4ppu4q1CJDrf264vUfnLlUbMlA4uNJPeIEA0ERs94mXaiiGDBMYkDFNsZlt3XIkkoJQ653y7wkBryC0rULIcZ6TQDxTG7ozVg8zkrLMdZtoE6Ac= ceph-5626d550-724a-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:36:50.034 INFO:teuthology.orchestra.run.smithi012.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOFQYcREEDvWG7GoLrTdkkx1RknDUvKqSb7EmGT+hJxyalMYinZLy0UpFlye2ULLWRc5/Rsk1NcXOuY+opI+x8KGs6R2acF/wHWnl0u79kUJEaDtmSZ5W7DK2NmWkjTgqTDxX47R4om4ziIlDUEjMmGCnHM2ISjEPVLB+2dip+nGVDhBApDq9dZ5rK0TGxqY2m6wdkxqXlMXl6qHOEa+V/oTle3pFnYPf1e1zrl5d+HxZ83plPNhtYFSe1HcU+Azqpr/HOILykKBtpVFKN4zy7cj8Q8yUJGRdA74FzDSW4HE4G2JF3cc4QjQ9jfnTFRV2NoP8MOmMmT00d6/xDBph7be9LMF1JHkvV3aUhxjOvE5ShtlEv5PwxwGHqzfdiUiQjDLFCPf/3btToVSJ4ppu4q1CJDrf264vUfnLlUbMlA4uNJPeIEA0ERs94mXaiiGDBMYkDFNsZlt3XIkkoJQ653y7wkBryC0rULIcZ6TQDxTG7ozVg8zkrLMdZtoE6Ac= ceph-5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:36:50.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-14T03:36:50.197 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:49 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/977366360' entity='client.admin' 2024-09-14T03:36:50.267 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:53.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:53.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:53.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi001", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:36:53.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:53.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:53.141 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:52 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:54.154 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-14T03:36:54.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-14T03:36:54.357 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:54 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2986359750' entity='client.admin' 2024-09-14T03:36:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:56.759 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi012 2024-09-14T03:36:56.759 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:36:56.759 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/etc/ceph/ceph.conf 2024-09-14T03:36:56.782 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:36:56.782 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:36:56.841 INFO:tasks.cephadm:Adding host smithi012 to orchestrator... 2024-09-14T03:36:56.841 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch host add smithi012 2024-09-14T03:36:57.049 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:36:57.258 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:57.259 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:57 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:36:58.169 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:58 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:36:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:36:59 smithi001 ceph-mon[29360]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi012", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:01 smithi001 ceph-mon[29360]: Deploying cephadm binary to smithi012 2024-09-14T03:37:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:01 smithi001 ceph-mon[29360]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:01.990 INFO:teuthology.orchestra.run.smithi001.stdout:Added host 'smithi012' with addr '172.21.15.12' 2024-09-14T03:37:02.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-14T03:37:02.773 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:03.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:03.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: Added host smithi012 2024-09-14T03:37:03.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:03.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:03.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:03.127 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:02 smithi001 ceph-mon[29360]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:04.421 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:37:04.421 INFO:teuthology.orchestra.run.smithi001.stdout:[{"addr": "172.21.15.1", "hostname": "smithi001", "labels": [], "status": ""}, {"addr": "172.21.15.12", "hostname": "smithi012", "labels": [], "status": ""}] 2024-09-14T03:37:04.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:04 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:04.976 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-14T03:37:04.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-14T03:37:05.178 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:05.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:05 smithi001 ceph-mon[29360]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:37:05.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:05 smithi001 ceph-mon[29360]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi012", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:07 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1290212826' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:37:07.979 INFO:teuthology.orchestra.run.smithi001.stderr:adjusted tunables profile to default 2024-09-14T03:37:08.292 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:08 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:37:08.292 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:08 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:37:08.292 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:08 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1290212826' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:37:08.292 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:08 smithi001 ceph-mon[29360]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:08.619 INFO:tasks.cephadm:Adding mon.a on smithi001 2024-09-14T03:37:08.619 INFO:tasks.cephadm:Adding mon.c on smithi001 2024-09-14T03:37:08.619 INFO:tasks.cephadm:Adding mon.b on smithi012 2024-09-14T03:37:08.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b' 2024-09-14T03:37:08.822 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:09 smithi001 ceph-mon[29360]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:10.513 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled mon update... 2024-09-14T03:37:11.104 DEBUG:teuthology.orchestra.run.smithi001:mon.c> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.c.service 2024-09-14T03:37:11.108 DEBUG:teuthology.orchestra.run.smithi012:mon.b> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.b.service 2024-09-14T03:37:11.110 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:37:11.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:37:11.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:11.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:11.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: Saving service mon spec with placement smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b;count:3 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:11.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:11 smithi001 ceph-mon[29360]: Deploying daemon mon.b on smithi012 2024-09-14T03:37:11.867 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:13 smithi001 ceph-mon[29360]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:14.298 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:37:14.298 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":1,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:35:47.568727Z","created":"2024-09-14T03:35:47.568727Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:3300","nonce":0},{"type":"v1","addr":"172.21.15.1:6789","nonce":0}]},"addr":"172.21.15.1:6789/0","public_addr":"172.21.15.1:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-14T03:37:14.298 INFO:teuthology.orchestra.run.smithi012.stderr:dumped monmap epoch 1 2024-09-14T03:37:14.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:14 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/37738006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:15.960 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:15 smithi012 podman[33602]: 2024-09-14 03:37:15.800857175 +0000 UTC m=+0.016416466 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T03:37:15.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:15 smithi001 ceph-mon[29360]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 podman[33602]: 2024-09-14 03:37:16.047227937 +0000 UTC m=+0.262787230 container init 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 podman[33602]: 2024-09-14 03:37:16.052353162 +0000 UTC m=+0.267912459 container start 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable), process ceph-mon, pid 7 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: pidfile_write: ignore empty --pid-file 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: load: jerasure load: lrc 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:37:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Git sha 0 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Compile date 2024-09-13 16:45:57 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: DB SUMMARY 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: DB Session ID: 3YZP5ACFP8MJZTURW2KL 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:37:16.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.env: 0x55f6d88e8bc0 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.info_log: 0x55f6d92f4be0 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:37:16.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.statistics: (nil) 2024-09-14T03:37:16.350 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:37:16.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.use_fsync: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:37:16.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.db_log_dir: 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.wal_dir: 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:37:16.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.write_buffer_manager: 0x55f6d9302320 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.unordered_write: 0 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:37:16.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.row_cache: None 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.wal_filter: None 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.wal_compression: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:37:16.355 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:37:16.356 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_open_files: -1 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:37:16.357 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Compression algorithms supported: 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kZSTD supported: 0 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:37:16.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.merge_operator: 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_filter: None 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f6d92f47e0) 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_type: 0 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: data_block_index_type: 0 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_shortening: 1 2024-09-14T03:37:16.359 INFO:journalctl@ceph.mon.b.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: checksum: 4 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: no_block_cache: 0 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache: 0x55f6d92e9350 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_options: 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: capacity : 536870912 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: num_shard_bits : 4 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: strict_capacity_limit : 0 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_compressed: (nil) 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: persistent_cache: (nil) 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_size: 4096 2024-09-14T03:37:16.360 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_size_deviation: 10 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_restart_interval: 16 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_block_restart_interval: 1 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: metadata_block_size: 4096 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: partition_filters: 0 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: use_delta_encoding: 1 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: filter_policy: bloomfilter 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: whole_key_filtering: 1 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: verify_compression: 0 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: format_version: 5 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: enable_index_compression: 1 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_align: 0 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: max_auto_readahead_size: 262144 2024-09-14T03:37:16.361 INFO:journalctl@ceph.mon.b.smithi012.stdout: prepopulate_block_cache: 0 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression: NoCompression 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.num_levels: 7 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:37:16.362 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:37:16.363 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:37:16.364 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:37:16.365 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:37:16.366 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:37:16.367 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.ttl: 2592000 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:37:16.368 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e2f3c8a5-7108-4746-90b3-36ffe31dde19 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285036080219, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285036080627, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1648, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285036, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e2f3c8a5-7108-4746-90b3-36ffe31dde19", "db_session_id": "3YZP5ACFP8MJZTURW2KL", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285036080733, "job": 1, "event": "recovery_finished"} 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f6d9314e00 2024-09-14T03:37:16.369 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: DB pointer 0x55f6d9408000 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** DB Stats ** 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:37:16.370 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: L0 1/0 1.61 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: Sum 1/0 1.61 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:37:16.371 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Block cache BinnedLRUCache@0x55f6d92e9350#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.372 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: using public_addr v2:172.21.15.12:0/0 -> [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: starting mon.b rank -1 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(???) e0 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).mds e1 new map 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-14T03:37:16.373 INFO:journalctl@ceph.mon.b.smithi012.stdout: e1 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout: legacy client fscid: -1 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout: No filesystems configured 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:16.374 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:16.375 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:16.375 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:16.375 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:16.376 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:16.376 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: Saving service mon spec with placement smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b;count:3 2024-09-14T03:37:16.376 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:16.376 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: Deploying daemon mon.b on smithi012 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/37738006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 ceph-mon[33640]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 bash[33602]: 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c 2024-09-14T03:37:16.377 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:16 smithi012 systemd[1]: Started Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:37:16.565 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:37:21.429 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:37:21.429 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":2,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:37:16.105217Z","created":"2024-09-14T03:35:47.568727Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:3300","nonce":0},{"type":"v1","addr":"172.21.15.1:6789","nonce":0}]},"addr":"172.21.15.1:6789/0","public_addr":"172.21.15.1:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-14T03:37:21.430 INFO:teuthology.orchestra.run.smithi012.stderr:dumped monmap epoch 2 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: mon.a calling monitor election 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: mon.b calling monitor election 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: monmap e2: 2 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: fsmap 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: mgrmap e13: y(active, since 40s) 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:21.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:21 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: mon.a calling monitor election 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: mon.b calling monitor election 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: monmap e2: 2 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: fsmap 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: mgrmap e13: y(active, since 40s) 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: overall HEALTH_OK 2024-09-14T03:37:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:21 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:22 smithi012 ceph-mon[33640]: Deploying daemon mon.c on smithi001 2024-09-14T03:37:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:22 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/448851998' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:22 smithi001 ceph-mon[29360]: Deploying daemon mon.c on smithi001 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:22 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/448851998' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:22.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:23.020 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:37:23.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:37:23.222 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:37:23.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:23 smithi012 ceph-mon[33640]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:23.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:23 smithi001 ceph-mon[29360]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:23.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:23 smithi001 systemd[1]: Starting Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 podman[40934]: 2024-09-14 03:37:23.982198107 +0000 UTC m=+0.016452942 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 podman[40934]: 2024-09-14 03:37:24.080371548 +0000 UTC m=+0.114626382 container create 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 podman[40934]: 2024-09-14 03:37:24.282844234 +0000 UTC m=+0.317099056 container init 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 podman[40934]: 2024-09-14 03:37:24.287993459 +0000 UTC m=+0.322248284 container start 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable), process ceph-mon, pid 7 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pidfile_write: ignore empty --pid-file 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: load: jerasure load: lrc 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:37:24.336 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Git sha 0 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Compile date 2024-09-13 16:45:57 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: DB SUMMARY 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: DB Session ID: U8HPWXFBQZT07RR59M9R 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-14T03:37:24.337 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.env: 0x55c073313bc0 2024-09-14T03:37:24.339 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.info_log: 0x55c073bf2be0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.statistics: (nil) 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.use_fsync: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:37:24.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.db_log_dir: 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.wal_dir: 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:37:24.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.write_buffer_manager: 0x55c073c00320 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.unordered_write: 0 2024-09-14T03:37:24.342 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.row_cache: None 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.wal_filter: None 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.wal_compression: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:37:24.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:37:24.344 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_open_files: -1 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Compression algorithms supported: 2024-09-14T03:37:24.345 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kZSTD supported: 0 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.merge_operator: 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_filter: None 2024-09-14T03:37:24.346 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c073bf27e0) 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_type: 0 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: data_block_index_type: 0 2024-09-14T03:37:24.347 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_shortening: 1 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: checksum: 4 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: no_block_cache: 0 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache: 0x55c073be7350 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_options: 2024-09-14T03:37:24.348 INFO:journalctl@ceph.mon.c.smithi001.stdout: capacity : 536870912 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: num_shard_bits : 4 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: strict_capacity_limit : 0 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_compressed: (nil) 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: persistent_cache: (nil) 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_size: 4096 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_size_deviation: 10 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_restart_interval: 16 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_block_restart_interval: 1 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: metadata_block_size: 4096 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: partition_filters: 0 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: use_delta_encoding: 1 2024-09-14T03:37:24.349 INFO:journalctl@ceph.mon.c.smithi001.stdout: filter_policy: bloomfilter 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: whole_key_filtering: 1 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: verify_compression: 0 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: format_version: 5 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: enable_index_compression: 1 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_align: 0 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: max_auto_readahead_size: 262144 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: prepopulate_block_cache: 0 2024-09-14T03:37:24.350 INFO:journalctl@ceph.mon.c.smithi001.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:37:24.351 INFO:journalctl@ceph.mon.c.smithi001.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:37:24.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:37:24.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:37:24.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression: NoCompression 2024-09-14T03:37:24.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.num_levels: 7 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:37:24.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:37:24.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:37:24.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:37:24.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:37:24.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:37:24.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:37:24.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:37:24.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:37:24.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:37:24.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:37:24.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:37:24.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:37:24.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:37:24.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:37:24.359 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:37:24.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:37:24.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:37:24.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:37:24.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:37:24.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:37:24.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:37:24.362 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:37:24.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:37:24.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:37:24.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:37:24.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.ttl: 2592000 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:37:24.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:37:24.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 23c7004b-485e-4e17-9495-92dd0d4b99b5 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285044315458, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-14T03:37:24.366 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285044315873, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285044, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "23c7004b-485e-4e17-9495-92dd0d4b99b5", "db_session_id": "U8HPWXFBQZT07RR59M9R", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285044315975, "job": 1, "event": "recovery_finished"} 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c073c12e00 2024-09-14T03:37:24.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: DB pointer 0x55c073d06000 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** DB Stats ** 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:24.368 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:37:24.369 INFO:journalctl@ceph.mon.c.smithi001.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:37:24.370 INFO:journalctl@ceph.mon.c.smithi001.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:37:24.371 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: Block cache BinnedLRUCache@0x55c073be7350#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:37:24.372 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: using public_addrv [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: starting mon.c rank -1 at public addrs [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] at bind addrs [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(???) e0 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).mds e1 new map 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-14T03:37:24.373 INFO:journalctl@ceph.mon.c.smithi001.stdout: e1 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout: legacy client fscid: -1 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout: No filesystems configured 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:37:24.374 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:37:24.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:24.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:24.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:37:24.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: Saving service mon spec with placement smithi001:172.21.15.1=a;smithi001:[v2:172.21.15.1:3301,v1:172.21.15.1:6790]=c;smithi012:172.21.15.12=b;count:3 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:24.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: Deploying daemon mon.b on smithi012 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/37738006' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.a calling monitor election 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.b calling monitor election 2024-09-14T03:37:24.380 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.380 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.380 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.380 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:37:24.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: monmap e2: 2 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:24.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: fsmap 2024-09-14T03:37:24.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mgrmap e13: y(active, since 40s) 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: overall HEALTH_OK 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: Deploying daemon mon.c on smithi001 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/448851998' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:24.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 ceph-mon[40972]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:37:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 bash[40934]: 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e 2024-09-14T03:37:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:24 smithi001 systemd[1]: Started Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:37:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: mon.a calling monitor election 2024-09-14T03:37:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: mon.b calling monitor election 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: mon.c calling monitor election 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: monmap e3: 3 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],c=[v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: fsmap 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: mgrmap e13: y(active, since 48s) 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: overall HEALTH_OK 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: mon.a calling monitor election 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: mon.b calling monitor election 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: mon.c calling monitor election 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: monmap e3: 3 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],c=[v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: fsmap 2024-09-14T03:37:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: mgrmap e13: y(active, since 48s) 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: overall HEALTH_OK 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.725 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:37:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: mon.a calling monitor election 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: mon.b calling monitor election 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: mon.c calling monitor election 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: monmap e3: 3 mons at {a=[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0],b=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],c=[v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: fsmap 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: mgrmap e13: y(active, since 48s) 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:29.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:30.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:30 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:30 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:30 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:37:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:30 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.004 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:37:31.004 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":3,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:37:24.339528Z","created":"2024-09-14T03:35:47.568727Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:3300","nonce":0},{"type":"v1","addr":"172.21.15.1:6789","nonce":0}]},"addr":"172.21.15.1:6789/0","public_addr":"172.21.15.1:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:3301","nonce":0},{"type":"v1","addr":"172.21.15.1:6790","nonce":0}]},"addr":"172.21.15.1:6790/0","public_addr":"172.21.15.1:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-14T03:37:31.004 INFO:teuthology.orchestra.run.smithi012.stderr:dumped monmap epoch 3 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:37:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/203463766' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:31.621 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-14T03:37:31.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:31.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/203463766' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/203463766' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:31.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:31.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:32.321 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:37:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:37:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[29360]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[29360]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[29360]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[40972]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[40972]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:37:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:33 smithi001 ceph-mon[40972]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:33 smithi012 ceph-mon[33640]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:37:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:33 smithi012 ceph-mon[33640]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:37:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:33 smithi012 ceph-mon[33640]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:35.082 INFO:teuthology.orchestra.run.smithi001.stdout:# minimal ceph.conf for 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:35.082 INFO:teuthology.orchestra.run.smithi001.stdout:[global] 2024-09-14T03:37:35.083 INFO:teuthology.orchestra.run.smithi001.stdout: fsid = 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:35.083 INFO:teuthology.orchestra.run.smithi001.stdout: mon_host = [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] 2024-09-14T03:37:35.633 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.633 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.633 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3741714980' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:37:35.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:35.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:35.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:37:35.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:35.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:35 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3741714980' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.655 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-14T03:37:35.655 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:37:35.656 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:37:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:37:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:35 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3741714980' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:35.725 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:37:35.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:37:35.797 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:37:35.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:37:35.827 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:37:35.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:37:35.894 INFO:tasks.cephadm:Adding mgr.y on smithi001 2024-09-14T03:37:35.894 INFO:tasks.cephadm:Adding mgr.x on smithi012 2024-09-14T03:37:35.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi001=y;smithi012=x' 2024-09-14T03:37:36.126 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:36 smithi012 ceph-mon[33640]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[29360]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:37.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:36 smithi001 ceph-mon[40972]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:37.777 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled mgr update... 2024-09-14T03:37:38.324 DEBUG:teuthology.orchestra.run.smithi012:mgr.x> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service 2024-09-14T03:37:38.327 DEBUG:tasks.cephadm:set 0 configs 2024-09-14T03:37:38.327 INFO:tasks.cephadm:Deploying OSDs... 2024-09-14T03:37:38.327 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:37:38.327 DEBUG:teuthology.orchestra.run.smithi001:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:37:38.348 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:37:38.348 DEBUG:teuthology.orchestra.run.smithi001:> stat /dev/vg_nvme/lv_1 2024-09-14T03:37:38.414 INFO:teuthology.orchestra.run.smithi001.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:37:38.414 INFO:teuthology.orchestra.run.smithi001.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:38.414 INFO:teuthology.orchestra.run.smithi001.stdout:Device: 5h/5d Inode: 979 Links: 1 2024-09-14T03:37:38.414 INFO:teuthology.orchestra.run.smithi001.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:38.415 INFO:teuthology.orchestra.run.smithi001.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:38.415 INFO:teuthology.orchestra.run.smithi001.stdout:Access: 2024-09-14 03:37:23.801232795 +0000 2024-09-14T03:37:38.415 INFO:teuthology.orchestra.run.smithi001.stdout:Modify: 2024-09-14 03:33:01.991547412 +0000 2024-09-14T03:37:38.415 INFO:teuthology.orchestra.run.smithi001.stdout:Change: 2024-09-14 03:33:01.991547412 +0000 2024-09-14T03:37:38.415 INFO:teuthology.orchestra.run.smithi001.stdout: Birth: 2024-09-14 03:33:01.991547412 +0000 2024-09-14T03:37:38.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:37:38.484 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records in 2024-09-14T03:37:38.484 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records out 2024-09-14T03:37:38.484 INFO:teuthology.orchestra.run.smithi001.stderr:512 bytes copied, 0.000132608 s, 3.9 MB/s 2024-09-14T03:37:38.486 DEBUG:teuthology.orchestra.run.smithi001:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:37:38.546 DEBUG:teuthology.orchestra.run.smithi001:> stat /dev/vg_nvme/lv_2 2024-09-14T03:37:38.604 INFO:teuthology.orchestra.run.smithi001.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:37:38.604 INFO:teuthology.orchestra.run.smithi001.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Device: 5h/5d Inode: 966 Links: 1 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Access: 2024-09-14 03:37:23.801232795 +0000 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Modify: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout:Change: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.605 INFO:teuthology.orchestra.run.smithi001.stdout: Birth: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:37:38.672 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records in 2024-09-14T03:37:38.672 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records out 2024-09-14T03:37:38.672 INFO:teuthology.orchestra.run.smithi001.stderr:512 bytes copied, 0.000109359 s, 4.7 MB/s 2024-09-14T03:37:38.673 DEBUG:teuthology.orchestra.run.smithi001:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:37:38.733 DEBUG:teuthology.orchestra.run.smithi001:> stat /dev/vg_nvme/lv_3 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Access: 2024-09-14 03:37:23.802232782 +0000 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Modify: 2024-09-14 03:33:01.992547479 +0000 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout:Change: 2024-09-14 03:33:01.992547479 +0000 2024-09-14T03:37:38.792 INFO:teuthology.orchestra.run.smithi001.stdout: Birth: 2024-09-14 03:33:01.992547479 +0000 2024-09-14T03:37:38.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:37:38.859 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records in 2024-09-14T03:37:38.859 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records out 2024-09-14T03:37:38.860 INFO:teuthology.orchestra.run.smithi001.stderr:512 bytes copied, 0.000116779 s, 4.4 MB/s 2024-09-14T03:37:38.861 DEBUG:teuthology.orchestra.run.smithi001:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:37:38.920 DEBUG:teuthology.orchestra.run.smithi001:> stat /dev/vg_nvme/lv_4 2024-09-14T03:37:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:37:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Device: 5h/5d Inode: 955 Links: 1 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Access: 2024-09-14 03:37:23.802232782 +0000 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Modify: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout:Change: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: Birth: 2024-09-14 03:33:01.990547344 +0000 2024-09-14T03:37:38.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:37:39.041 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi001=y;smithi012=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.041 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: Saving service mgr spec with placement smithi001=y;smithi012=x;count:2 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: Deploying daemon mgr.x on smithi012 2024-09-14T03:37:39.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[29360]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi001=y;smithi012=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: Saving service mgr spec with placement smithi001=y;smithi012=x;count:2 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: Deploying daemon mgr.x on smithi012 2024-09-14T03:37:39.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:38 smithi001 ceph-mon[40972]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:39.044 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records in 2024-09-14T03:37:39.044 INFO:teuthology.orchestra.run.smithi001.stderr:1+0 records out 2024-09-14T03:37:39.044 INFO:teuthology.orchestra.run.smithi001.stderr:512 bytes copied, 0.000117674 s, 4.4 MB/s 2024-09-14T03:37:39.045 DEBUG:teuthology.orchestra.run.smithi001:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi001=y;smithi012=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: Saving service mgr spec with placement smithi001=y;smithi012=x;count:2 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:37:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: Deploying daemon mgr.x on smithi012 2024-09-14T03:37:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:38 smithi012 ceph-mon[33640]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:39.103 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:37:39.103 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:37:39.121 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:37:39.121 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_1 2024-09-14T03:37:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 979 Links: 1 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-14 03:37:15.596981399 +0000 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:37:39.243 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-14T03:37:39.243 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-14T03:37:39.243 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000210789 s, 2.4 MB/s 2024-09-14T03:37:39.244 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:37:39.301 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_2 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 962 Links: 1 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-14 03:37:15.596981399 +0000 2024-09-14T03:37:39.358 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-14 03:33:02.096035803 +0000 2024-09-14T03:37:39.359 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-14 03:33:02.096035803 +0000 2024-09-14T03:37:39.359 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-14 03:33:02.096035803 +0000 2024-09-14T03:37:39.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:37:39.422 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-14T03:37:39.423 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-14T03:37:39.423 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000215528 s, 2.4 MB/s 2024-09-14T03:37:39.424 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:37:39.481 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_3 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 984 Links: 1 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-14 03:37:15.597981391 +0000 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.537 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.538 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:37:39.675 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-14T03:37:39.676 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-14T03:37:39.676 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000209299 s, 2.4 MB/s 2024-09-14T03:37:39.677 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:37:39.694 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_4 2024-09-14T03:37:39.757 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:37:39.757 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-14 03:37:39.626784703 +0000 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.758 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-14 03:33:02.098035982 +0000 2024-09-14T03:37:39.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:37:39.862 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-14T03:37:39.862 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-14T03:37:39.862 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000182559 s, 2.8 MB/s 2024-09-14T03:37:39.863 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:37:39.880 INFO:tasks.cephadm:Deploying osd.0 on smithi001 with /dev/vg_nvme/lv_4... 2024-09-14T03:37:39.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:37:40.089 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:40.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:39 smithi012 systemd[1]: Starting Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:37:40.369 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 podman[35712]: 2024-09-14 03:37:40.002747363 +0000 UTC m=+0.015915743 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T03:37:40.369 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 podman[35712]: 2024-09-14 03:37:40.11859404 +0000 UTC m=+0.131762405 container create 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-14T03:37:40.369 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 podman[35712]: 2024-09-14 03:37:40.337554599 +0000 UTC m=+0.350722971 container init 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-14T03:37:40.369 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 podman[35712]: 2024-09-14 03:37:40.342498903 +0000 UTC m=+0.355667268 container start 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-09-14T03:37:40.370 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-mgr[35750]: -- 172.21.15.12:0/4221974659 <== mon.1 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cda263cd00 con 0x55cda2611400 2024-09-14T03:37:40.689 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 bash[35712]: 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c 2024-09-14T03:37:40.689 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 systemd[1]: Started Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:37:40.689 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:40.466+0000 7fbe944f1040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:37:40.689 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:40.526+0000 7fbe944f1040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:37:40.689 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:40.686+0000 7fbe944f1040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:37:41.078 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:40.762+0000 7fbe944f1040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:37:41.079 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:40.930+0000 7fbe944f1040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:37:41.079 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:41.076+0000 7fbe944f1040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:37:41.672 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:41.411+0000 7fbe944f1040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:37:41.672 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:41.474+0000 7fbe944f1040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:37:41.672 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:41.669+0000 7fbe944f1040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:37:41.672 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.673 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:42.525 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:37:42.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi001:vg_nvme/lv_4 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:37:42.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[29360]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:37:42.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:42 smithi001 ceph-mon[40972]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:42.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:42.578+0000 7fbe944f1040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:37:42.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:42.647+0000 7fbe944f1040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:37:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:37:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:42 smithi012 ceph-mon[33640]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:43.289 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:37:43.668 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.398+0000 7fbe944f1040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:37:43.668 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.517+0000 7fbe944f1040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:37:43.954 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.665+0000 7fbe944f1040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:37:43.954 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.729+0000 7fbe944f1040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:37:43.954 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.881+0000 7fbe944f1040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:37:43.954 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:43.951+0000 7fbe944f1040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:37:44.245 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.013+0000 7fbe944f1040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:37:44.245 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.176+0000 7fbe944f1040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:37:44.245 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.243+0000 7fbe944f1040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:37:44.521 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.311+0000 7fbe944f1040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:37:44.521 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.396+0000 7fbe944f1040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:37:44.521 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.457+0000 7fbe944f1040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:37:44.521 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:44.518+0000 7fbe944f1040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:37:45.134 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.135 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:45.136 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.334 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.334 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:45.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:45.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:45.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:37:45.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:45.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:37:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:37:45.332+0000 7fbe944f1040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:37:46.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: Standby manager daemon x started 2024-09-14T03:37:46.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:37:46.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:37:46.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:37:46.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:37:46.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:46 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/2579776400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:37:46.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:37:46.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:37:46.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:46 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:47.139 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[29360]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:37:47.139 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:37:47.139 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[29360]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:47.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[40972]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:37:47.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:37:47.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:47 smithi001 ceph-mon[40972]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:47 smithi012 ceph-mon[33640]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:37:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:47 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:37:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:47 smithi012 ceph-mon[33640]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:48 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3130903993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:48 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:48 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]': finished 2024-09-14T03:37:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:48 smithi012 ceph-mon[33640]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:37:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3130903993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]': finished 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[29360]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:37:48.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3130903993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]: dispatch 2024-09-14T03:37:48.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86fadbf5-5c8f-4719-b817-f0ff28fb4caa"}]': finished 2024-09-14T03:37:48.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[40972]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:37:48.358 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:37:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:49 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3156910830' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:37:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:49 smithi012 ceph-mon[33640]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:49 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3156910830' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:37:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:49 smithi001 ceph-mon[29360]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:49 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3156910830' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:37:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:49 smithi001 ceph-mon[40972]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:51.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:51 smithi001 ceph-mon[29360]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:51.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:51 smithi001 ceph-mon[40972]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:51 smithi012 ceph-mon[33640]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:53 smithi012 ceph-mon[33640]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:53.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:53 smithi001 ceph-mon[29360]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:53.924 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:53 smithi001 ceph-mon[40972]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:54.829 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:37:54.829 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:54.829 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:54 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:37:54.829 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:54 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:54 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:37:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:54 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:37:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:55 smithi012 ceph-mon[33640]: Deploying daemon osd.0 on smithi001 2024-09-14T03:37:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:55 smithi012 ceph-mon[33640]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:55.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:55 smithi001 ceph-mon[29360]: Deploying daemon osd.0 on smithi001 2024-09-14T03:37:55.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:55 smithi001 ceph-mon[29360]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:55.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:55 smithi001 ceph-mon[40972]: Deploying daemon osd.0 on smithi001 2024-09-14T03:37:55.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:55 smithi001 ceph-mon[40972]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:56.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:56 smithi001 ceph-mon[29360]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:56.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:56 smithi001 ceph-mon[40972]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:56 smithi012 ceph-mon[33640]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[29360]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[40972]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:58.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:58.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:58.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:37:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:58 smithi012 ceph-mon[33640]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:37:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:37:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:37:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:37:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:01.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:01.821 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:01.856 INFO:teuthology.orchestra.run.smithi001.stdout:Created osd(s) 0 on host 'smithi001' 2024-09-14T03:38:02.464 DEBUG:teuthology.orchestra.run.smithi001:osd.0> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service 2024-09-14T03:38:02.470 INFO:tasks.cephadm:Deploying osd.1 on smithi001 with /dev/vg_nvme/lv_3... 2024-09-14T03:38:02.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[29360]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[29360]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[40972]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[40972]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:02 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:02 smithi012 ceph-mon[33640]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:02 smithi012 ceph-mon[33640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:38:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:02 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:02 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:02 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:03.252 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[29360]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:03.698 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:38:03.698 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:38:03.698 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:03.698 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.698 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-mon[40972]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: from='osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:03 smithi012 ceph-mon[33640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:03.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[45360]: 2024-09-14T03:38:03.776+0000 7f24dbdd5640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T03:38:03.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:03 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[45360]: 2024-09-14T03:38:03.778+0000 7f24d33df640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[29360]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[29360]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[29360]: from='osd.0 ' entity='osd.0' 2024-09-14T03:38:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[40972]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:04.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[40972]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:38:04.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:04 smithi001 ceph-mon[40972]: from='osd.0 ' entity='osd.0' 2024-09-14T03:38:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:04 smithi012 ceph-mon[33640]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:04 smithi012 ceph-mon[33640]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:38:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:04 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:04 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:04 smithi012 ceph-mon[33640]: from='osd.0 ' entity='osd.0' 2024-09-14T03:38:05.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:38:05.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:38:05.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:05.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039] boot 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039] boot 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:38:05.618 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:05 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: osd.0 [v2:172.21.15.1:6802/3758176039,v1:172.21.15.1:6803/3758176039] boot 2024-09-14T03:38:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:38:05.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:05 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:38:05.879 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:38:05.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi001:vg_nvme/lv_3 2024-09-14T03:38:06.425 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:07 smithi001 ceph-mon[29360]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:07 smithi001 ceph-mon[29360]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:38:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:07 smithi001 ceph-mon[40972]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:07.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:07 smithi001 ceph-mon[40972]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:38:07.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:07 smithi012 ceph-mon[33640]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:07.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:07 smithi012 ceph-mon[33640]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi001 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi001 to 16068M 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:09.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi001 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi001 to 16068M 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:09.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:09.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:09.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:09.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi001 2024-09-14T03:38:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi001 to 16068M 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:09.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:10 smithi012 ceph-mon[33640]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:10 smithi001 ceph-mon[29360]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:10.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:10 smithi001 ceph-mon[40972]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: pgmap v44: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1637595028' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]': finished 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: pgmap v44: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1637595028' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]': finished 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:38:11.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:11 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: pgmap v44: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1637595028' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]: dispatch 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbf0be8f-6a29-4e63-8dca-4d830fcd7d47"}]': finished 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:38:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:11 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:12 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/707536973' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:12 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/707536973' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:12 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/707536973' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:13 smithi012 ceph-mon[33640]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:13.719 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:13 smithi001 ceph-mon[29360]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:13.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:13 smithi001 ceph-mon[40972]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:15.614 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:15 smithi001 ceph-mon[29360]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:15.614 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:15 smithi001 ceph-mon[40972]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:15 smithi012 ceph-mon[33640]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:17 smithi012 ceph-mon[33640]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:17 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:38:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:17 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:17.927 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[29360]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:17.927 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:38:17.927 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:17.927 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[40972]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:17.927 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:38:17.928 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:17 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:18.629 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:18 smithi001 ceph-mon[29360]: Deploying daemon osd.1 on smithi001 2024-09-14T03:38:18.629 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:18 smithi001 ceph-mon[29360]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:18.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:18 smithi001 ceph-mon[40972]: Deploying daemon osd.1 on smithi001 2024-09-14T03:38:18.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:18 smithi001 ceph-mon[40972]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:18 smithi012 ceph-mon[33640]: Deploying daemon osd.1 on smithi001 2024-09-14T03:38:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:18 smithi012 ceph-mon[33640]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:21 smithi012 ceph-mon[33640]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:21 smithi001 ceph-mon[29360]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:21.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:21 smithi001 ceph-mon[40972]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:22.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:23.795 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:23 smithi001 ceph-mon[29360]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:23.795 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:23 smithi001 ceph-mon[40972]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:23 smithi012 ceph-mon[33640]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:24.949 INFO:teuthology.orchestra.run.smithi001.stdout:Created osd(s) 1 on host 'smithi001' 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:38:25.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:25.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.558 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.579 DEBUG:teuthology.orchestra.run.smithi001:osd.1> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.1.service 2024-09-14T03:38:25.581 INFO:tasks.cephadm:Deploying osd.2 on smithi001 with /dev/vg_nvme/lv_2... 2024-09-14T03:38:25.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:25.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:38:25.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:25.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:25.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:26.234 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:26 smithi012 ceph-mon[33640]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:38:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:26 smithi012 ceph-mon[33640]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:38:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:26 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:26 smithi012 ceph-mon[33640]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:26.603 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[29360]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:38:26.603 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[29360]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[29360]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[40972]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[40972]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:26.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-mon[40972]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:26.604 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[51716]: 2024-09-14T03:38:26.315+0000 7fed9a313640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T03:38:26.604 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[51716]: 2024-09-14T03:38:26.318+0000 7fed9613c640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:27.510 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[29360]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[29360]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[29360]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[40972]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:27.511 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[40972]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:38:27.512 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.512 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.512 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:27 smithi001 ceph-mon[40972]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:27 smithi012 ceph-mon[33640]: from='osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:27 smithi012 ceph-mon[33640]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:38:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:27 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:27 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:27 smithi012 ceph-mon[33640]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858] boot 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:38:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858] boot 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:38:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:38:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:38:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: osd.1 [v2:172.21.15.1:6810/997975858,v1:172.21.15.1:6811/997975858] boot 2024-09-14T03:38:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:38:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi001 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi001 to 8034M 2024-09-14T03:38:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:29 smithi012 ceph-mon[33640]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:38:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi001 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi001 to 8034M 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[40972]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:38:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi001 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi001 to 8034M 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:29.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:29 smithi001 ceph-mon[29360]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:30.029 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:38:30.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi001:vg_nvme/lv_2 2024-09-14T03:38:30.252 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:31.777 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:31 smithi001 ceph-mon[29360]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:31.777 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:31 smithi001 ceph-mon[40972]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:31 smithi012 ceph-mon[33640]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:32.679 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:32 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:32 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:33.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:33 smithi001 ceph-mon[29360]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:33.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:33 smithi001 ceph-mon[29360]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:33.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:33 smithi001 ceph-mon[40972]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:33.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:33 smithi001 ceph-mon[40972]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:33 smithi012 ceph-mon[33640]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:33 smithi012 ceph-mon[33640]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:34 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]: dispatch 2024-09-14T03:38:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:34 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]': finished 2024-09-14T03:38:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:34 smithi012 ceph-mon[33640]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:38:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:34 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:34 smithi012 ceph-mon[33640]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:34.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]: dispatch 2024-09-14T03:38:34.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]': finished 2024-09-14T03:38:34.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[29360]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:38:34.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:34.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[29360]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:34.941 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]: dispatch 2024-09-14T03:38:34.941 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3261650190' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca27ddd4-8678-4b03-be41-f1600e007add"}]': finished 2024-09-14T03:38:34.941 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[40972]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:38:34.941 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:34.941 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:34 smithi001 ceph-mon[40972]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:35 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2114403703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:35.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:35 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2114403703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:35 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2114403703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:38:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:36 smithi012 ceph-mon[33640]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:36.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:36 smithi001 ceph-mon[40972]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:36 smithi001 ceph-mon[29360]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:39.670 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:39 smithi001 ceph-mon[29360]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:39.670 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:39 smithi001 ceph-mon[40972]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:39 smithi012 ceph-mon[33640]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:40 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:38:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:40 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:40.873 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:38:40.873 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:40 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:40.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:40 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:38:40.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:40 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:41.825 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:41 smithi001 ceph-mon[29360]: Deploying daemon osd.2 on smithi001 2024-09-14T03:38:41.826 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:41 smithi001 ceph-mon[29360]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:41.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:41 smithi001 ceph-mon[40972]: Deploying daemon osd.2 on smithi001 2024-09-14T03:38:41.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:41 smithi001 ceph-mon[40972]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:41 smithi012 ceph-mon[33640]: Deploying daemon osd.2 on smithi001 2024-09-14T03:38:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:41 smithi012 ceph-mon[33640]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:42.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:42 smithi001 ceph-mon[29360]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:42.725 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:42 smithi001 ceph-mon[40972]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:42 smithi012 ceph-mon[33640]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:45 smithi012 ceph-mon[33640]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[40972]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[29360]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:45.864 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:45.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:45.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:46 smithi001 ceph-mon[29360]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:46 smithi001 ceph-mon[40972]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:46 smithi012 ceph-mon[33640]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:47.764 INFO:teuthology.orchestra.run.smithi001.stdout:Created osd(s) 2 on host 'smithi001' 2024-09-14T03:38:48.470 DEBUG:teuthology.orchestra.run.smithi001:osd.2> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service 2024-09-14T03:38:48.486 INFO:tasks.cephadm:Deploying osd.3 on smithi001 with /dev/vg_nvme/lv_1... 2024-09-14T03:38:48.486 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:38:48.854 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:49.067 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.067 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[29360]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[29360]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[29360]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[40972]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.068 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[40972]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.069 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:48 smithi001 ceph-mon[40972]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:48 smithi012 ceph-mon[33640]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:48 smithi012 ceph-mon[33640]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:38:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:48 smithi012 ceph-mon[33640]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[29360]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[29360]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[29360]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[29360]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[40972]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[40972]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:38:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:49.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[40972]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:49.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:49 smithi001 ceph-mon[40972]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:49 smithi012 ceph-mon[33640]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:38:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:49 smithi012 ceph-mon[33640]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:38:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:49 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:49 smithi012 ceph-mon[33640]: from='osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:49 smithi012 ceph-mon[33640]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:38:50.351 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[58000]: 2024-09-14T03:38:50.053+0000 7fde699bc640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T03:38:50.351 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[58000]: 2024-09-14T03:38:50.056+0000 7fde617dd640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:38:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:50.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:50.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: from='osd.2 ' entity='osd.2' 2024-09-14T03:38:50.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[29360]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:50.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:50.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: from='osd.2 ' entity='osd.2' 2024-09-14T03:38:50.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:50 smithi001 ceph-mon[40972]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: from='osd.2 ' entity='osd.2' 2024-09-14T03:38:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:50 smithi012 ceph-mon[33640]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:38:51.444 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:38:51.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi001:vg_nvme/lv_1 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830] boot 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:38:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:38:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830] boot 2024-09-14T03:38:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:38:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:51 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: osd.2 [v2:172.21.15.1:6818/4255524830,v1:172.21.15.1:6819/4255524830] boot 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:38:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:51 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:38:52.342 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:38:53.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[29360]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:38:53.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[29360]: pgmap v75: 0 pgs: ; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:53.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[40972]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:38:53.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[40972]: pgmap v75: 0 pgs: ; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:53.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:53 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:53 smithi012 ceph-mon[33640]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:38:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:53 smithi012 ceph-mon[33640]: pgmap v75: 0 pgs: ; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:53 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:54.166 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:54.167 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[29360]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:38:54.167 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:54.167 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:54.167 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[40972]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:38:54.167 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:54 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:54 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:54 smithi012 ceph-mon[33640]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:38:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:54 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:38:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:38:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi001 2024-09-14T03:38:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi001 to 5356M 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62056]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:55.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62056]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:55.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62056]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:55.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62056]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi001 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi001 to 5356M 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:55.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi001 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi001 to 5356M 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:38:55.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:55.766 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62180]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:38:55.767 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62180]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:55.767 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62180]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:55.767 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:38:55 smithi001 sudo[62180]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:38:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:56.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62220]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:56.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62220]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:56.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62220]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:56.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62220]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:56.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:56.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:38:56.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:56.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi001:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:38:56.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:38:56.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:38:56.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:56.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:38:56.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:38:56.799 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62231]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:38:56.799 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62231]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:56.799 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62231]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:56.799 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62231]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:57.078 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62274]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:57.078 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62274]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:57.078 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62274]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:57.078 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:56 smithi001 sudo[62274]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:57 smithi012 ceph-mon[33640]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:57.388 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:57 smithi001 ceph-mon[29360]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:57.388 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:57 smithi001 ceph-mon[40972]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 88 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:38:57.720 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62323]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:38:57.721 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62323]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:57.721 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62323]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:57.721 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62323]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:58.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62327]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62327]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62327]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:57 smithi001 sudo[62327]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:58.551 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:58 smithi001 ceph-mon[40972]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:38:58.551 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:58 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:38:58.551 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 sudo[62330]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:38:58.551 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 sudo[62330]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:58.552 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 sudo[62330]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:58.552 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 sudo[62330]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:58.552 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 ceph-mon[29360]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:38:58.552 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:38:58 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:38:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 ceph-mon[33640]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:38:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:38:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 sudo[36400]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 sudo[36400]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 sudo[36400]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:58 smithi012 sudo[36400]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:59 smithi012 sudo[36403]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:38:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:59 smithi012 sudo[36403]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:59 smithi012 sudo[36403]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:38:59 smithi012 sudo[36403]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:38:59.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:59 smithi001 sudo[62598]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:38:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:59 smithi001 sudo[62598]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:38:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:59 smithi001 sudo[62598]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:38:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:38:59 smithi001 sudo[62598]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:39:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]': finished 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/738657438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 sudo[62785]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 sudo[62785]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 sudo[62785]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 sudo[62785]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]': finished 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/738657438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:00.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]: dispatch 2024-09-14T03:39:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3237395351' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7004b178-c338-490e-a6f2-fb8510c93c29"}]': finished 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/738657438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:39:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:39:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[40972]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[40972]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[29360]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:01 smithi001 ceph-mon[29360]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:01 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:39:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:01 smithi012 ceph-mon[33640]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:39:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:01 smithi012 ceph-mon[33640]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 89 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:03.843 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:03 smithi001 ceph-mon[29360]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:03.844 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:03 smithi001 ceph-mon[40972]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:03.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:03 smithi012 ceph-mon[33640]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:04 smithi012 ceph-mon[33640]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-14T03:39:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:04 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:39:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:04 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:04 smithi012 ceph-mon[33640]: Deploying daemon osd.3 on smithi001 2024-09-14T03:39:04.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[29360]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-14T03:39:04.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:39:04.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:04.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[29360]: Deploying daemon osd.3 on smithi001 2024-09-14T03:39:04.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[40972]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-14T03:39:04.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:39:04.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:04.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:04 smithi001 ceph-mon[40972]: Deploying daemon osd.3 on smithi001 2024-09-14T03:39:07.805 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:07 smithi001 ceph-mon[29360]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-14T03:39:07.805 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:07 smithi001 ceph-mon[40972]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-14T03:39:07.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:07 smithi012 ceph-mon[33640]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-14T03:39:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:08 smithi012 ceph-mon[33640]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:39:08.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:08 smithi001 ceph-mon[29360]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:39:08.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:08 smithi001 ceph-mon[40972]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:39:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:09.877 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:09.878 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:09.878 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:09.878 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:09.878 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:09.878 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:10 smithi012 ceph-mon[33640]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:10 smithi001 ceph-mon[40972]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:10 smithi001 ceph-mon[29360]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:12.742 INFO:teuthology.orchestra.run.smithi001.stdout:Created osd(s) 3 on host 'smithi001' 2024-09-14T03:39:13.319 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.320 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.321 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:39:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:39:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:13.433 DEBUG:teuthology.orchestra.run.smithi001:osd.3> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.3.service 2024-09-14T03:39:13.436 INFO:tasks.cephadm:Deploying osd.4 on smithi012 with /dev/vg_nvme/lv_4... 2024-09-14T03:39:13.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:39:13.645 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[29360]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[29360]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[29360]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[40972]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:39:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[40972]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:39:14.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:14.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-mon[40972]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:39:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:14 smithi012 ceph-mon[33640]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:39:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:14 smithi012 ceph-mon[33640]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:39:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:14 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:14 smithi012 ceph-mon[33640]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]: dispatch 2024-09-14T03:39:14.670 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[64135]: 2024-09-14T03:39:14.290+0000 7f355721c640 -1 osd.3 0 waiting for initial osdmap 2024-09-14T03:39:14.670 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 03:39:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[64135]: 2024-09-14T03:39:14.294+0000 7f354e826640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:15 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:39:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]}]': finished 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: from='osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347]' entity='osd.3' 2024-09-14T03:39:15.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 90 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:39:15.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:15 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:15.894 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:39:15.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_4 2024-09-14T03:39:16.116 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347] boot 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347] boot 2024-09-14T03:39:16.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:16.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:16 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: osd.3 [v2:172.21.15.1:6826/2316309347,v1:172.21.15.1:6827/2316309347] boot 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:16.495 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:16 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:17.428 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:17 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi001 2024-09-14T03:39:17.428 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:17 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi001 to 4017M 2024-09-14T03:39:17.428 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:17 smithi012 ceph-mon[33640]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:39:17.428 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:17 smithi012 ceph-mon[33640]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi001 2024-09-14T03:39:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi001 to 4017M 2024-09-14T03:39:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[29360]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:39:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[29360]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi001 2024-09-14T03:39:17.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi001 to 4017M 2024-09-14T03:39:17.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[40972]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:39:17.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:17 smithi001 ceph-mon[40972]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:18.450 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:18.450 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:18.450 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:18.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:19.559 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:19 smithi012 ceph-mon[33640]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:19.560 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:19 smithi012 ceph-mon[33640]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:19 smithi001 ceph-mon[29360]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:19 smithi001 ceph-mon[29360]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:19 smithi001 ceph-mon[40972]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:19 smithi001 ceph-mon[40972]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:20 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/1359735550' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:20 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:20 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]': finished 2024-09-14T03:39:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:20 smithi012 ceph-mon[33640]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:39:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:20 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/1359735550' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]': finished 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[29360]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/1359735550' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]: dispatch 2024-09-14T03:39:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40284648-ea27-4db8-977d-cfd9e41c3b7f"}]': finished 2024-09-14T03:39:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[40972]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:39:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:20 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:21.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:21 smithi012 ceph-mon[33640]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:21.358 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:21 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/1217630103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:21 smithi001 ceph-mon[29360]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:21 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/1217630103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:21 smithi001 ceph-mon[40972]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:21 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/1217630103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:22 smithi012 ceph-mon[33640]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:22 smithi001 ceph-mon[29360]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:22.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:22 smithi001 ceph-mon[40972]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:24.936 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:24 smithi012 ceph-mon[33640]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:24 smithi001 ceph-mon[29360]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:24 smithi001 ceph-mon[40972]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:26.311 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:39:26.311 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:39:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:39:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:27.257 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:26 smithi012 ceph-mon[33640]: Deploying daemon osd.4 on smithi012 2024-09-14T03:39:27.257 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:26 smithi012 ceph-mon[33640]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:26 smithi001 ceph-mon[29360]: Deploying daemon osd.4 on smithi012 2024-09-14T03:39:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:26 smithi001 ceph-mon[29360]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:26 smithi001 ceph-mon[40972]: Deploying daemon osd.4 on smithi012 2024-09-14T03:39:27.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:26 smithi001 ceph-mon[40972]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:28.910 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:28 smithi012 ceph-mon[33640]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:28 smithi001 ceph-mon[29360]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:28 smithi001 ceph-mon[40972]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:30 smithi012 ceph-mon[33640]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:30 smithi001 ceph-mon[29360]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:30 smithi001 ceph-mon[40972]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:31 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:31 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:32 smithi012 ceph-mon[33640]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:32 smithi001 ceph-mon[29360]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:32 smithi001 ceph-mon[40972]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:33.989 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 4 on host 'smithi012' 2024-09-14T03:39:34.625 DEBUG:teuthology.orchestra.run.smithi012:osd.4> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.4.service 2024-09-14T03:39:34.627 INFO:tasks.cephadm:Deploying osd.5 on smithi012 with /dev/vg_nvme/lv_3... 2024-09-14T03:39:34.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:39:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:34 smithi012 ceph-mon[33640]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:35.347 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[29360]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:39:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:34 smithi001 ceph-mon[40972]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:36.221 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-mon[33640]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:39:36.221 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-mon[33640]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:39:36.222 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:36.222 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-mon[33640]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.222 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-mon[33640]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[29360]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[29360]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[29360]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[29360]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[40972]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:39:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[40972]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:39:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[40972]: from='osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:36 smithi001 ceph-mon[40972]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:36.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T03:39:36.460+0000 7f200f4c1640 -1 osd.4 0 waiting for initial osdmap 2024-09-14T03:39:36.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 03:39:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T03:39:36.465+0000 7f20072e2640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:39:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:39:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:39:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: from='osd.4 ' entity='osd.4' 2024-09-14T03:39:37.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:37 smithi012 ceph-mon[33640]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: from='osd.4 ' entity='osd.4' 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[29360]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:39:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:39:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: from='osd.4 ' entity='osd.4' 2024-09-14T03:39:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:37 smithi001 ceph-mon[40972]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 120 MiB used, 357 GiB / 358 GiB avail 2024-09-14T03:39:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:38 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:39:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:38 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:39:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:38 smithi012 ceph-mon[33640]: osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415] boot 2024-09-14T03:39:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:38 smithi012 ceph-mon[33640]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:39:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:38 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[40972]: osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415] boot 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[40972]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[29360]: osd.4 [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415] boot 2024-09-14T03:39:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[29360]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:39:38.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:38 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:39:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:39:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi012 2024-09-14T03:39:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 17092M 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:39 smithi012 ceph-mon[33640]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi012 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 17092M 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[40972]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi012 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 17092M 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:39 smithi001 ceph-mon[29360]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:39:39.881 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:39:39.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_3 2024-09-14T03:39:40.106 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:39:41.301 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:41 smithi012 ceph-mon[33640]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:39:41.301 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:41 smithi012 ceph-mon[33640]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:41 smithi001 ceph-mon[40972]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:39:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:41 smithi001 ceph-mon[40972]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:41 smithi001 ceph-mon[29360]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:39:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:41 smithi001 ceph-mon[29360]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:42.304 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:42 smithi012 ceph-mon[33640]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:39:42.304 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:42.304 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:42.304 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:42 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[29360]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:39:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:42.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[40972]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:39:42.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:39:42.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:39:42.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:42 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:43.186 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:43 smithi012 ceph-mon[33640]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:43.187 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:43 smithi012 ceph-mon[33640]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:43 smithi001 ceph-mon[40972]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:43 smithi001 ceph-mon[40972]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:43 smithi001 ceph-mon[29360]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:43 smithi001 ceph-mon[29360]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:44 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/929977073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:44 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:44 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]': finished 2024-09-14T03:39:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:44 smithi012 ceph-mon[33640]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:39:44.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/929977073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]': finished 2024-09-14T03:39:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[40972]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:39:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/929977073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]: dispatch 2024-09-14T03:39:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6"}]': finished 2024-09-14T03:39:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[29360]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:39:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:45 smithi012 ceph-mon[33640]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:39:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:45 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/3803663504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:45 smithi001 ceph-mon[29360]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:39:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:45 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/3803663504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:45 smithi001 ceph-mon[40972]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:39:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:45 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/3803663504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:39:46.879 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:46 smithi012 ceph-mon[33640]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:39:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:46 smithi001 ceph-mon[29360]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:39:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:46 smithi001 ceph-mon[40972]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 148 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:39:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:48 smithi012 ceph-mon[33640]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:39:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:48 smithi001 ceph-mon[29360]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:39:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:48 smithi001 ceph-mon[40972]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:39:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:39:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:50 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:39:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:50 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:39:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:50 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:51.226 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:51 smithi012 ceph-mon[33640]: Deploying daemon osd.5 on smithi012 2024-09-14T03:39:51.226 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:51 smithi012 ceph-mon[33640]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:39:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:51 smithi001 ceph-mon[29360]: Deploying daemon osd.5 on smithi012 2024-09-14T03:39:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:51 smithi001 ceph-mon[29360]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:39:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:51 smithi001 ceph-mon[40972]: Deploying daemon osd.5 on smithi012 2024-09-14T03:39:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:51 smithi001 ceph-mon[40972]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:39:52.910 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:52 smithi012 ceph-mon[33640]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:39:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:52 smithi001 ceph-mon[29360]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:39:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:52 smithi001 ceph-mon[40972]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:39:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:54 smithi001 ceph-mon[29360]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:54 smithi001 ceph-mon[40972]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:54 smithi012 ceph-mon[33640]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:55.665 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:55.665 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:55.665 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:55 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:55 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:56.865 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:56 smithi012 ceph-mon[33640]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:56 smithi001 ceph-mon[29360]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:56 smithi001 ceph-mon[40972]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:57.906 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 5 on host 'smithi012' 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:58.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:58 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:58.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:58.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.480 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.480 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:39:58.480 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:58.480 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.480 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:58 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:39:58.502 DEBUG:teuthology.orchestra.run.smithi012:osd.5> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5.service 2024-09-14T03:39:58.504 INFO:tasks.cephadm:Deploying osd.6 on smithi012 with /dev/vg_nvme/lv_2... 2024-09-14T03:39:58.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:39:59.331 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-mon[33640]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[29360]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:39:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:39:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:39:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: from='osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:39:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:39:59 smithi001 ceph-mon[40972]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 147 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:39:59.846 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[45362]: 2024-09-14T03:39:59.655+0000 7f670ab96640 -1 osd.5 0 waiting for initial osdmap 2024-09-14T03:39:59.846 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 03:39:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[45362]: 2024-09-14T03:39:59.660+0000 7f67029b7640 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: from='osd.5 ' entity='osd.5' 2024-09-14T03:40:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[29360]: overall HEALTH_OK 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: from='osd.5 ' entity='osd.5' 2024-09-14T03:40:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:00 smithi001 ceph-mon[40972]: overall HEALTH_OK 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: from='osd.5 ' entity='osd.5' 2024-09-14T03:40:00.483 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:00 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883] boot 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:40:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[29360]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883] boot 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:40:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:01 smithi001 ceph-mon[40972]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:40:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:40:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: osd.5 [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883] boot 2024-09-14T03:40:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:40:01.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:40:01.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:40:01.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:01 smithi012 ceph-mon[33640]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:02.593 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:02 smithi012 ceph-mon[33640]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:40:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:02 smithi001 ceph-mon[29360]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:40:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:02 smithi001 ceph-mon[40972]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi012 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 8546M 2024-09-14T03:40:03.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.590 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:03.590 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:03.590 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.590 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:03.590 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:03 smithi012 ceph-mon[33640]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:40:03.591 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:40:03.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_2 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi012 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 8546M 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[29360]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi012 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 8546M 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:03.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:03 smithi001 ceph-mon[40972]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:40:03.813 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:04 smithi012 ceph-mon[33640]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:40:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:04 smithi001 ceph-mon[29360]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:40:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:04 smithi001 ceph-mon[40972]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:40:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:05 smithi012 ceph-mon[33640]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:05 smithi001 ceph-mon[40972]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:05 smithi001 ceph-mon[29360]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:06.488 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:06 smithi012 ceph-mon[33640]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:06.488 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:06 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:06.488 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:06 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:06.488 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:06 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[29360]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[40972]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:06 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:07.406 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:07 smithi012 ceph-mon[33640]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 175 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:07 smithi001 ceph-mon[29360]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 175 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:07 smithi001 ceph-mon[40972]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 175 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/1645258993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]': finished 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:40:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:08.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:08 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/1299042138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: Cluster is now healthy 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/1645258993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]': finished 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/1299042138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: Cluster is now healthy 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/1645258993' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]: dispatch 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "786e0de6-b9fc-4b30-bd1f-a888694aeeab"}]': finished 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:08 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/1299042138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:09 smithi012 ceph-mon[33640]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-14T03:40:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:09 smithi001 ceph-mon[40972]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-14T03:40:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:09 smithi001 ceph-mon[29360]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-14T03:40:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:10 smithi012 ceph-mon[33640]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:40:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:10 smithi001 ceph-mon[40972]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:40:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:10 smithi001 ceph-mon[29360]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:40:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:12 smithi012 ceph-mon[33640]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-14T03:40:12.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:12 smithi001 ceph-mon[29360]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-14T03:40:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:12 smithi001 ceph-mon[40972]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-14T03:40:13.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:40:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:13 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:40:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:13 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:14.036 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:40:14.037 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:13 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:14 smithi012 ceph-mon[33640]: Deploying daemon osd.6 on smithi012 2024-09-14T03:40:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:14 smithi012 ceph-mon[33640]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:14 smithi001 ceph-mon[40972]: Deploying daemon osd.6 on smithi012 2024-09-14T03:40:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:14 smithi001 ceph-mon[40972]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:14 smithi001 ceph-mon[29360]: Deploying daemon osd.6 on smithi012 2024-09-14T03:40:15.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:14 smithi001 ceph-mon[29360]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:16.920 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:16 smithi012 ceph-mon[33640]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:16 smithi001 ceph-mon[29360]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:16 smithi001 ceph-mon[40972]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:18 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:18 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:19.812 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:19 smithi012 ceph-mon[33640]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:19 smithi001 ceph-mon[40972]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:19 smithi001 ceph-mon[29360]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:20.761 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:20 smithi012 ceph-mon[33640]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:20 smithi001 ceph-mon[29360]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:20 smithi001 ceph-mon[40972]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:21.348 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 6 on host 'smithi012' 2024-09-14T03:40:21.989 DEBUG:teuthology.orchestra.run.smithi012:osd.6> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.6.service 2024-09-14T03:40:21.991 INFO:tasks.cephadm:Deploying osd.7 on smithi012 with /dev/vg_nvme/lv_1... 2024-09-14T03:40:21.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:40:22.241 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:22.586 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.586 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:22 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:22.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:22 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:23.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:40:23.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:40:23.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:23.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-mon[33640]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:40:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[29360]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: from='osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:23.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:23 smithi001 ceph-mon[40972]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:24.096 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T03:40:23.835+0000 7fd16445a640 -1 osd.6 0 waiting for initial osdmap 2024-09-14T03:40:24.097 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 03:40:23 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T03:40:23.842+0000 7fd15ba64640 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[29360]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[29360]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[29360]: from='osd.6 ' entity='osd.6' 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[40972]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[40972]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:40:24.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:24 smithi001 ceph-mon[40972]: from='osd.6 ' entity='osd.6' 2024-09-14T03:40:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:24 smithi012 ceph-mon[33640]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:24 smithi012 ceph-mon[33640]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:40:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:24 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:24 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:24 smithi012 ceph-mon[33640]: from='osd.6 ' entity='osd.6' 2024-09-14T03:40:24.949 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-14T03:40:24.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_1 2024-09-14T03:40:25.412 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370] boot 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:40:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:25 smithi012 ceph-mon[33640]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370] boot 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[29360]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 174 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: osd.6 [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370] boot 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:40:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:25 smithi001 ceph-mon[40972]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:40:27.561 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:27 smithi012 ceph-mon[33640]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:40:27.561 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:27 smithi012 ceph-mon[33640]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:27 smithi001 ceph-mon[29360]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:40:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:27 smithi001 ceph-mon[29360]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:27 smithi001 ceph-mon[40972]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:40:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:27 smithi001 ceph-mon[40972]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:28 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:40:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:28.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:28.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:28.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:28 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:29.471 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:29 smithi012 ceph-mon[33640]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:29.472 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:29 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi012 2024-09-14T03:40:29.472 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:29 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 5697M 2024-09-14T03:40:29.472 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:29 smithi012 ceph-mon[33640]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:40:29.472 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:29 smithi012 ceph-mon[33640]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[29360]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi012 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 5697M 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[29360]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[29360]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[40972]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi012 2024-09-14T03:40:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 5697M 2024-09-14T03:40:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[40972]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:40:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:29 smithi001 ceph-mon[40972]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:30.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:30 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/4140240248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:30 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:30 smithi012 ceph-mon[33640]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]': finished 2024-09-14T03:40:30.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:30 smithi012 ceph-mon[33640]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:40:30.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:30 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/4140240248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[29360]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]': finished 2024-09-14T03:40:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[29360]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:40:30.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:30.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/4140240248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]: dispatch 2024-09-14T03:40:30.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[40972]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "409b8e01-c0ad-4f5c-b67a-95b914426b8a"}]': finished 2024-09-14T03:40:30.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[40972]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:40:30.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:30 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:31 smithi012 ceph-mon[33640]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-14T03:40:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:31 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/876252849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:31 smithi001 ceph-mon[29360]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-14T03:40:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:31 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/876252849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:31 smithi001 ceph-mon[40972]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 201 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-14T03:40:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:31 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/876252849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:32 smithi001 ceph-mon[29360]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:32 smithi001 ceph-mon[40972]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:33.043 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:32 smithi012 ceph-mon[33640]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:34.724 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:34 smithi012 ceph-mon[33640]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T03:40:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:34 smithi001 ceph-mon[29360]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T03:40:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:34 smithi001 ceph-mon[40972]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T03:40:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:40:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:36 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:40:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:36 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:40:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:36 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:37.311 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:37 smithi012 ceph-mon[33640]: Deploying daemon osd.7 on smithi012 2024-09-14T03:40:37.312 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:37 smithi012 ceph-mon[33640]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-14T03:40:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:37 smithi001 ceph-mon[29360]: Deploying daemon osd.7 on smithi012 2024-09-14T03:40:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:37 smithi001 ceph-mon[29360]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-14T03:40:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:37 smithi001 ceph-mon[40972]: Deploying daemon osd.7 on smithi012 2024-09-14T03:40:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:37 smithi001 ceph-mon[40972]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-14T03:40:38.833 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:38 smithi012 ceph-mon[33640]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:38 smithi001 ceph-mon[29360]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:38 smithi001 ceph-mon[40972]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:40:40.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:40 smithi012 ceph-mon[33640]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-14T03:40:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:40 smithi001 ceph-mon[29360]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-14T03:40:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:40 smithi001 ceph-mon[40972]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-14T03:40:41.750 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:41.750 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:41.750 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:41 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:41 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:42 smithi012 ceph-mon[33640]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:42.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:42 smithi001 ceph-mon[29360]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:42.872 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:42 smithi001 ceph-mon[40972]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:44.288 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 7 on host 'smithi012' 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:44 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.868 DEBUG:teuthology.orchestra.run.smithi012:osd.7> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.7.service 2024-09-14T03:40:44.870 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-14T03:40:44.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-14T03:40:44.900 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.900 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.900 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:44.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:40:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:44 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:45.093 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:45.797 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:45.797 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:40:45.797 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:40:45.797 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:45.797 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:45.798 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:45 smithi012 ceph-mon[33640]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[40972]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:45.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:40:45.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:40:45.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:45.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: from='osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:45.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:45 smithi001 ceph-mon[29360]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-14T03:40:46.601 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 03:40:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T03:40:46.438+0000 7f77ad3fb640 -1 osd.7 0 waiting for initial osdmap 2024-09-14T03:40:46.601 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 03:40:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T03:40:46.447+0000 7f77a521c640 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: from='osd.7 ' entity='osd.7' 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[29360]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.272 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: from='osd.7 ' entity='osd.7' 2024-09-14T03:40:47.272 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:47 smithi001 ceph-mon[40972]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: from='osd.7 ' entity='osd.7' 2024-09-14T03:40:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:47 smithi012 ceph-mon[33640]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 202 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:40:47.953 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726285246,"num_in_osds":8,"osd_in_since":1726285229,"num_remapped_pgs":1} 2024-09-14T03:40:47.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:40:48.197 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: purged_snaps scrub starts 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: purged_snaps scrub ok 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999] boot 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/161786229' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: purged_snaps scrub starts 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: purged_snaps scrub ok 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999] boot 2024-09-14T03:40:48.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/161786229' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:48.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:48 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: purged_snaps scrub starts 2024-09-14T03:40:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: purged_snaps scrub ok 2024-09-14T03:40:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: osd.7 [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999] boot 2024-09-14T03:40:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/161786229' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:48 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:49 smithi012 ceph-mon[33640]: Detected new or changed devices on smithi012 2024-09-14T03:40:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:49 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 4273M 2024-09-14T03:40:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:49 smithi012 ceph-mon[33640]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:40:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:49 smithi012 ceph-mon[33640]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[29360]: Detected new or changed devices on smithi012 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 4273M 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[29360]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[29360]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[40972]: Detected new or changed devices on smithi012 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 4273M 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[40972]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:40:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:49 smithi001 ceph-mon[40972]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:50 smithi001 ceph-mon[29360]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:40:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:50 smithi001 ceph-mon[40972]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:40:50.254 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:40:50.255 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":58,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T03:40:50.017151+0000","last_up_change":"2024-09-14T03:40:46.998732+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6803","nonce":3758176039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6805","nonce":3758176039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6809","nonce":3758176039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6807","nonce":3758176039}]},"public_addr":"172.21.15.1:6803/3758176039","cluster_addr":"172.21.15.1:6805/3758176039","heartbeat_back_addr":"172.21.15.1:6809/3758176039","heartbeat_front_addr":"172.21.15.1:6807/3758176039","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6811","nonce":997975858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6813","nonce":997975858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6817","nonce":997975858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6815","nonce":997975858}]},"public_addr":"172.21.15.1:6811/997975858","cluster_addr":"172.21.15.1:6813/997975858","heartbeat_back_addr":"172.21.15.1:6817/997975858","heartbeat_front_addr":"172.21.15.1:6815/997975858","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6819","nonce":4255524830}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6821","nonce":4255524830}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6825","nonce":4255524830}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6823","nonce":4255524830}]},"public_addr":"172.21.15.1:6819/4255524830","cluster_addr":"172.21.15.1:6821/4255524830","heartbeat_back_addr":"172.21.15.1:6825/4255524830","heartbeat_front_addr":"172.21.15.1:6823/4255524830","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6827","nonce":2316309347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6829","nonce":2316309347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6833","nonce":2316309347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6831","nonce":2316309347}]},"public_addr":"172.21.15.1:6827/2316309347","cluster_addr":"172.21.15.1:6829/2316309347","heartbeat_back_addr":"172.21.15.1:6833/2316309347","heartbeat_front_addr":"172.21.15.1:6831/2316309347","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:50 smithi012 ceph-mon[33640]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:40:50.953 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-14T03:38:52.963897+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-09-14T03:40:50.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-14T03:40:51.158 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[29360]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3629045957' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[29360]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[40972]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3629045957' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:51 smithi001 ceph-mon[40972]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:51 smithi012 ceph-mon[33640]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:40:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:51 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3629045957' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:40:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:51 smithi012 ceph-mon[33640]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 412 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:40:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:52 smithi012 ceph-mon[33640]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:40:52.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:52 smithi001 ceph-mon[29360]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:40:52.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:52 smithi001 ceph-mon[40972]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:40:53.051 INFO:teuthology.orchestra.run.smithi001.stdout:pg_num: 1 2024-09-14T03:40:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:53 smithi012 ceph-mon[33640]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:40:53.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:53 smithi001 ceph-mon[29360]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:40:53.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:53 smithi001 ceph-mon[40972]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-14T03:40:53.753 INFO:tasks.cephadm:Adding prometheus.a on smithi012 2024-09-14T03:40:53.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi012=a' 2024-09-14T03:40:53.970 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:54.093 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:54 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/195259437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:40:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:54 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/195259437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:40:54.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:54 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/195259437' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:40:55.068 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:55 smithi012 ceph-mon[33640]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:55 smithi001 ceph-mon[29360]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:55 smithi001 ceph-mon[40972]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:40:55.735 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled prometheus update... 2024-09-14T03:40:56.380 DEBUG:teuthology.orchestra.run.smithi012:prometheus.a> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-14T03:40:56.383 INFO:tasks.cephadm:Adding node-exporter.a on smithi001 2024-09-14T03:40:56.383 INFO:tasks.cephadm:Adding node-exporter.b on smithi012 2024-09-14T03:40:56.383 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi001=a;smithi012=b' 2024-09-14T03:40:56.637 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:57.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:57.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: Saving service prometheus spec with placement smithi012=a;count:1 2024-09-14T03:40:57.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:57.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:57.020 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:57.020 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.020 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:40:57.020 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:56 smithi012 ceph-mon[33640]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: Saving service prometheus spec with placement smithi012=a;count:1 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:40:57.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[29360]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: Saving service prometheus spec with placement smithi012=a;count:1 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:40:57.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:56 smithi001 ceph-mon[40972]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T03:40:58.811 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled node-exporter update... 2024-09-14T03:40:58.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:58 smithi001 ceph-mon[29360]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-14T03:40:58.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:58 smithi001 ceph-mon[40972]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-14T03:40:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:58 smithi012 ceph-mon[33640]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-14T03:40:59.655 DEBUG:teuthology.orchestra.run.smithi001:node-exporter.a> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-14T03:40:59.658 DEBUG:teuthology.orchestra.run.smithi012:node-exporter.b> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-14T03:40:59.661 INFO:tasks.cephadm:Adding alertmanager.a on smithi001 2024-09-14T03:40:59.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi001=a' 2024-09-14T03:40:59.903 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[40972]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi001=a;smithi012=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[40972]: Saving service node-exporter spec with placement smithi001=a;smithi012=b;count:2 2024-09-14T03:40:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:40:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[29360]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi001=a;smithi012=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[29360]: Saving service node-exporter spec with placement smithi001=a;smithi012=b;count:2 2024-09-14T03:40:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:40:59 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:00.024 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:59 smithi012 ceph-mon[33640]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi001=a;smithi012=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:00.024 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:59 smithi012 ceph-mon[33640]: Saving service node-exporter spec with placement smithi001=a;smithi012=b;count:2 2024-09-14T03:41:00.024 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:40:59 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:00 smithi012 ceph-mon[33640]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:41:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:00 smithi001 ceph-mon[29360]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:41:01.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:00 smithi001 ceph-mon[40972]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:41:02.333 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled alertmanager update... 2024-09-14T03:41:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:03 smithi012 ceph-mon[33640]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi001=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:03 smithi012 ceph-mon[33640]: Saving service alertmanager spec with placement smithi001=a;count:1 2024-09-14T03:41:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:03 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:03 smithi012 ceph-mon[33640]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[29360]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi001=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[29360]: Saving service alertmanager spec with placement smithi001=a;count:1 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[29360]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[40972]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi001=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[40972]: Saving service alertmanager spec with placement smithi001=a;count:1 2024-09-14T03:41:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:03 smithi001 ceph-mon[40972]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-14T03:41:04.277 DEBUG:teuthology.orchestra.run.smithi001:alertmanager.a> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-14T03:41:04.280 INFO:tasks.cephadm:Adding grafana.a on smithi012 2024-09-14T03:41:04.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi012=a' 2024-09-14T03:41:04.660 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:41:04.954 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:04 smithi012 ceph-mon[33640]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:04.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:04 smithi001 ceph-mon[29360]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:04 smithi001 ceph-mon[40972]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:06 smithi012 ceph-mon[33640]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:06 smithi001 ceph-mon[29360]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:06 smithi001 ceph-mon[40972]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:08.286 INFO:teuthology.orchestra.run.smithi012.stdout:Scheduled grafana update... 2024-09-14T03:41:09.066 DEBUG:teuthology.orchestra.run.smithi012:grafana.a> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service 2024-09-14T03:41:09.068 INFO:tasks.cephadm:Setting up client nodes... 2024-09-14T03:41:09.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:41:09.284 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:09 smithi012 ceph-mon[33640]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:09 smithi012 ceph-mon[33640]: Saving service grafana spec with placement smithi012=a;count:1 2024-09-14T03:41:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:09 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:09 smithi012 ceph-mon[33640]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[29360]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[29360]: Saving service grafana spec with placement smithi012=a;count:1 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[29360]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[40972]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[40972]: Saving service grafana spec with placement smithi012=a;count:1 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:09.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:09 smithi001 ceph-mon[40972]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:10.242 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 systemd[1]: Starting Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:41:10.493 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 podman[63317]: 2024-09-14 03:41:10.144739177 +0000 UTC m=+0.016974552 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:41:10.493 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 podman[63317]: 2024-09-14 03:41:10.245782843 +0000 UTC m=+0.118018208 container create 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:10.493 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 podman[63317]: 2024-09-14 03:41:10.456592102 +0000 UTC m=+0.328827468 container init 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:10.493 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 podman[63317]: 2024-09-14 03:41:10.461645897 +0000 UTC m=+0.333881263 container start 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.486Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.486Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.487Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi012 (none))" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.487Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.487Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.488Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.488Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.489Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.489Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.492Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.492Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.92µs 2024-09-14T03:41:10.494 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.492Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:41:10.495 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.492Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-14T03:41:10.495 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.492Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.811µs wal_replay_duration=277.602µs wbl_replay_duration=130ns total_replay_duration=317.469µs 2024-09-14T03:41:10.747 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[29360]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:10.747 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[40972]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.748 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:10 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:41:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-mon[33640]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' 2024-09-14T03:41:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:41:10.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.493Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.493Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.493Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.513Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.337485ms db_storage=894ns remote_storage=1.329µs web_handler=401ns query_engine=678ns scrape=529.407µs scrape_sd=80.783µs notify=731ns notify_sd=1.304µs rules=19.297022ms tracing=5.983µs 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.513Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:41:10.513Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 bash[63317]: 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef 2024-09-14T03:41:10.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:41:10 smithi012 systemd[1]: Started Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:41:11.218 INFO:teuthology.orchestra.run.smithi001.stdout:[client.0] 2024-09-14T03:41:11.218 INFO:teuthology.orchestra.run.smithi001.stdout: key = AQDXBeVmgim+DBAA9F5tYEhymQ7XYa2BsRpfKw== 2024-09-14T03:41:11.799 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2510222237' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.799 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.799 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:11.799 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[29360]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:41:11.799 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[29360]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2510222237' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[40972]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-mon[40972]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setuser ceph since I am not root 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setgroup ceph since I am not root 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:11.738+0000 7fe4394c4040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:41:11.800 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:11.797+0000 7fe4394c4040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:41:11.933 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T03:41:11.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-14T03:41:11.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-14T03:41:11.958 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: ignoring --setuser ceph since I am not root 2024-09-14T03:41:11.958 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: ignoring --setgroup ceph since I am not root 2024-09-14T03:41:11.958 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mgr[35750]: -- 172.21.15.12:0/3493440463 <== mon.0 v2:172.21.15.1:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fde5052d00 con 0x55fde5027400 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:11.745+0000 7f2334feb040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:11.803+0000 7f2334feb040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2510222237' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mon[33640]: from='mgr.14148 172.21.15.1:0/2108704777' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:41:11.959 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-mon[33640]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:41:11.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:41:12.184 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:41:12.186 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:11.965+0000 7fe4394c4040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:41:12.186 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.034+0000 7fe4394c4040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:41:12.300 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:11.957+0000 7f2334feb040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:41:12.301 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.027+0000 7f2334feb040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:41:12.301 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.190+0000 7f2334feb040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:41:12.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.184+0000 7fe4394c4040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:41:12.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.330+0000 7fe4394c4040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:41:12.562 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.340+0000 7f2334feb040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:41:12.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.668+0000 7f2334feb040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:41:12.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.731+0000 7f2334feb040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:41:12.905 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.651+0000 7fe4394c4040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:41:12.905 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.714+0000 7fe4394c4040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:41:13.306 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:12.923+0000 7f2334feb040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:41:13.459 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:12.903+0000 7fe4394c4040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:41:14.079 INFO:teuthology.orchestra.run.smithi012.stdout:[client.1] 2024-09-14T03:41:14.079 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDaBeVmT+91BBAADSnLG/+EFhwtNJYU+PI9gQ== 2024-09-14T03:41:14.128 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:13.876+0000 7f2334feb040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:41:14.128 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:13.946+0000 7f2334feb040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:41:14.128 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:13.832+0000 7fe4394c4040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:41:14.128 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:13.902+0000 7fe4394c4040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:41:14.406 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-mon[33640]: from='client.? 172.21.15.12:0/1531841437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.406 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.407 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:14.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[40972]: from='client.? 172.21.15.12:0/1531841437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:14.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[29360]: from='client.? 172.21.15.12:0/1531841437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:41:14.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:41:14.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:14.653+0000 7fe4394c4040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:41:14.760 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-14T03:41:14.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-14T03:41:14.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-14T03:41:14.800 INFO:tasks.cephadm:Creating RBD pool 2024-09-14T03:41:14.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-14T03:41:14.973 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:14.700+0000 7f2334feb040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:41:14.973 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:14.824+0000 7f2334feb040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:41:15.013 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:15.130 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:14.773+0000 7fe4394c4040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:41:15.131 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:14.922+0000 7fe4394c4040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:41:15.131 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:14.995+0000 7fe4394c4040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:41:15.260 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:14.971+0000 7f2334feb040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:41:15.260 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.035+0000 7f2334feb040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:41:15.260 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.188+0000 7f2334feb040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:41:15.260 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.258+0000 7f2334feb040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:41:15.399 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.154+0000 7fe4394c4040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:41:15.399 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.225+0000 7fe4394c4040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:41:15.400 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.306+0000 7fe4394c4040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:41:15.551 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.320+0000 7f2334feb040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:41:15.551 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.482+0000 7f2334feb040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:41:15.551 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.550+0000 7f2334feb040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:41:15.698 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.474+0000 7fe4394c4040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:41:15.698 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.542+0000 7fe4394c4040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:41:15.699 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.611+0000 7fe4394c4040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:41:15.699 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.697+0000 7fe4394c4040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:41:15.828 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.618+0000 7f2334feb040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:41:15.828 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.704+0000 7f2334feb040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:41:15.828 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.765+0000 7f2334feb040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:41:15.828 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:15.826+0000 7f2334feb040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:41:15.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.758+0000 7fe4394c4040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:41:15.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:15.820+0000 7fe4394c4040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:41:16.847 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: Standby manager daemon x restarted 2024-09-14T03:41:16.847 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:41:16.847 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: Active manager daemon y restarted 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: Activating manager daemon y 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: mgrmap e17: y(active, starting, since 0.00738485s), standbys: x 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:16.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:41:16.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[40972]: Manager daemon y is now available 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: Standby manager daemon x restarted 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: Standby manager daemon x started 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: Active manager daemon y restarted 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: Activating manager daemon y 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: mgrmap e17: y(active, starting, since 0.00738485s), standbys: x 2024-09-14T03:41:16.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:41:16.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-mon[29360]: Manager daemon y is now available 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:16.652+0000 7fe4394c4040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:16] ENGINE Bus STARTING 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: CherryPy Checker: 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: The Application mounted at '' has an empty config. 2024-09-14T03:41:16.852 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:17.051 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:16.637+0000 7f2334feb040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:41:17.051 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: [14/Sep/2024:03:41:16] ENGINE Bus STARTING 2024-09-14T03:41:17.051 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: CherryPy Checker: 2024-09-14T03:41:17.051 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: The Application mounted at '' has an empty config. 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: [14/Sep/2024:03:41:16] ENGINE Serving on http://:::9283 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[35727]: [14/Sep/2024:03:41:16] ENGINE Bus STARTED 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: Standby manager daemon x restarted 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:41:17.052 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/3274044768' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: Active manager daemon y restarted 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: Activating manager daemon y 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: mgrmap e17: y(active, starting, since 0.00738485s), standbys: x 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:41:17.053 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:41:17.054 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:16 smithi012 ceph-mon[33640]: Manager daemon y is now available 2024-09-14T03:41:17.099 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:16] ENGINE Serving on http://:::9283 2024-09-14T03:41:17.099 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:16] ENGINE Bus STARTED 2024-09-14T03:41:17.710 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'rbd' created 2024-09-14T03:41:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1277055964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:17.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[40972]: mgrmap e18: y(active, since 1.0122s), standbys: x 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1277055964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:17.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:17 smithi001 ceph-mon[29360]: mgrmap e18: y(active, since 1.0122s), standbys: x 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1277055964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:41:18.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:17 smithi012 ceph-mon[33640]: mgrmap e18: y(active, since 1.0122s), standbys: x 2024-09-14T03:41:18.777 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[29360]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[29360]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[40972]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:41:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:18 smithi001 ceph-mon[40972]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:41:18.998 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:18 smithi012 ceph-mon[33640]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:18 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:41:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:18 smithi012 ceph-mon[33640]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:41:20.051 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:41:20.051 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:41:20.051 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.051 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.051 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 4017M 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[29360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 4017M 2024-09-14T03:41:20.054 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.054 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:19 smithi001 ceph-mon[40972]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 4017M 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:19 smithi012 ceph-mon[33640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:20.048+0000 7fe38d9ab640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:41:20] ENGINE Error in HTTPServer.serve 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: Traceback (most recent call last): 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._connections.run(self.expiration_interval) 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._run(expiration_interval) 2024-09-14T03:41:20.466 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: s = self.context.wrap_socket( 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: return self.sslsocket_class._create( 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self.do_handshake() 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: self._sslobj.do_handshake() 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:41:20.467 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:41:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:41:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: [14/Sep/2024:03:41:19] ENGINE Bus STARTING 2024-09-14T03:41:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: [14/Sep/2024:03:41:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:41:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: [14/Sep/2024:03:41:20] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: [14/Sep/2024:03:41:20] ENGINE Bus STARTED 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: [14/Sep/2024:03:41:20] ENGINE Error in HTTPServer.serve 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: Traceback (most recent call last): 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._run(expiration_interval) 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: s = self.context.wrap_socket( 2024-09-14T03:41:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: return self.sslsocket_class._create( 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: self.do_handshake() 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._sslobj.do_handshake() 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:20 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:41:19] ENGINE Bus STARTING 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:41:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:41:20] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:41:20] ENGINE Bus STARTED 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: [14/Sep/2024:03:41:20] ENGINE Error in HTTPServer.serve 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:41:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:41:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: [14/Sep/2024:03:41:19] ENGINE Bus STARTING 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: [14/Sep/2024:03:41:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: [14/Sep/2024:03:41:20] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: [14/Sep/2024:03:41:20] ENGINE Bus STARTED 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: [14/Sep/2024:03:41:20] ENGINE Error in HTTPServer.serve 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:41:21.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: self.do_handshake() 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:21.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:41:21.226 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:41:21.226 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.226 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:20 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:41:21.997 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1915583874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:21.998 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:41:21.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:21 smithi012 ceph-mon[33640]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:41:21] "GET /metrics HTTP/1.1" 200 23656 "" "Prometheus/2.43.0" 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1915583874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[40972]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1915583874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:41:22.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:41:22.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:21 smithi001 ceph-mon[29360]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:41:23.014 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:22 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:41:23.015 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:22 smithi012 ceph-mon[33640]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:41:23.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:22 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:41:23.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:22 smithi001 ceph-mon[29360]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:41:23.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:22 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:41:23.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:22 smithi001 ceph-mon[40972]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:41:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:23 smithi012 ceph-mon[33640]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:23 smithi012 ceph-mon[33640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:41:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:23 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:41:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:23 smithi012 ceph-mon[33640]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[29360]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[29360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[29360]: Cluster is now healthy 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[29360]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[40972]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[40972]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:41:24.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[40972]: Cluster is now healthy 2024-09-14T03:41:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:23 smithi001 ceph-mon[40972]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:41:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:24 smithi001 ceph-mon[29360]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:41:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:24 smithi001 ceph-mon[40972]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:41:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:24 smithi012 ceph-mon[33640]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:41:25.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-14T03:41:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:25 smithi012 ceph-mon[33640]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:41:26.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:25 smithi001 ceph-mon[29360]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:41:26.097 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:25 smithi001 ceph-mon[40972]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:41:26.566 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:27.892 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:27 smithi001 ceph-mon[29360]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:41:27.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:27 smithi001 ceph-mon[40972]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:41:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:27 smithi012 ceph-mon[33640]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:41:29.636 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:41:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:29 smithi012 ceph-mon[33640]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 295 B/s wr, 17 op/s 2024-09-14T03:41:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:29 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3689764267' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:41:30.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:29 smithi001 ceph-mon[29360]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 295 B/s wr, 17 op/s 2024-09-14T03:41:30.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:29 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3689764267' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:41:30.117 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:29 smithi001 ceph-mon[40972]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 295 B/s wr, 17 op/s 2024-09-14T03:41:30.117 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:29 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3689764267' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:41:30.721 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:30 smithi001 systemd[1]: Starting Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:41:31.086 INFO:teuthology.orchestra.run.smithi001.stdout:[{"version":25,"timestamp":"2024-09-14T03:41:19.606323+0000","name":"","changes":[{"name":"osd/host:smithi012/osd_memory_target","previous_value":"4480656486","new_value":"4212221030"}]}] 2024-09-14T03:41:31.087 INFO:tasks.ceph_manager:config epoch is 25 2024-09-14T03:41:31.087 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:41:31.087 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:41:31.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:41:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:31 smithi012 ceph-mon[33640]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:41:31.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-mon[29360]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:41:31.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-mon[40972]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:41:31.607 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 podman[72098]: 2024-09-14 03:41:31.234555356 +0000 UTC m=+0.019660758 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:41:31.607 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 podman[72098]: 2024-09-14 03:41:31.330683273 +0000 UTC m=+0.115788641 volume create 81d2a5c88bd4e0ae46621fba4c32c137b427c2411a2e22e0fddfdee53472d98c 2024-09-14T03:41:31.607 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 podman[72098]: 2024-09-14 03:41:31.406145852 +0000 UTC m=+0.191251221 container create ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:31.848 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:31.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:41:31] "GET /metrics HTTP/1.1" 200 28487 "" "Prometheus/2.43.0" 2024-09-14T03:41:31.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:31] ENGINE Bus STOPPING 2024-09-14T03:41:31.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 podman[72098]: 2024-09-14 03:41:31.709051484 +0000 UTC m=+0.494156850 container init ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 podman[72098]: 2024-09-14 03:41:31.714669997 +0000 UTC m=+0.499775363 container start ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.730Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.730Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.732Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.1 port=9094 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.733Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:41:31.972 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.758Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:41:31.973 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.758Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:41:31.973 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.762Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:41:31.973 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:31.762Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:41:31.973 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 bash[72098]: ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 2024-09-14T03:41:31.973 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:31 smithi001 systemd[1]: Started Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-14T03:41:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:32] ENGINE Bus STOPPED 2024-09-14T03:41:32.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:32] ENGINE Bus STARTING 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:32] ENGINE Serving on http://:::9283 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:41:32] ENGINE Bus STARTED 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:32.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:32.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[40972]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[40972]: Deploying daemon grafana.a on smithi012 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[29360]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:33.587 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-mon[29360]: Deploying daemon grafana.a on smithi012 2024-09-14T03:41:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:33 smithi012 ceph-mon[33640]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:41:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:33 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:41:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:33 smithi012 ceph-mon[33640]: Deploying daemon grafana.a on smithi012 2024-09-14T03:41:33.794 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:41:33.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:33.734Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000904823s 2024-09-14T03:41:34.578 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":20,"flags":0,"active_gid":24350,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6800","nonce":2630182969},{"type":"v1","addr":"172.21.15.1:6801","nonce":2630182969}]},"active_addr":"172.21.15.1:6801/2630182969","active_change":"2024-09-14T03:41:16.654863+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":739412744}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":416113000}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":3674234575}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":2817796360}]}]} 2024-09-14T03:41:34.583 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:41:34.584 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:41:34.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:41:34.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:34 smithi012 ceph-mon[33640]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-14T03:41:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:34 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1831409292' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:41:34.611 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:34 smithi001 ceph-mon[40972]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-14T03:41:34.611 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:34 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1831409292' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:41:34.611 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:34 smithi001 ceph-mon[29360]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-14T03:41:34.612 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:34 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1831409292' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:41:34.799 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:36.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:36 smithi001 ceph-mon[29360]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:36.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:36 smithi001 ceph-mon[40972]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:36.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:36 smithi012 ceph-mon[33640]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:37.039 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:41:37.039 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":67,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T03:41:30.324273+0000","last_up_change":"2024-09-14T03:40:46.998732+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6803","nonce":3758176039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6805","nonce":3758176039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6809","nonce":3758176039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6807","nonce":3758176039}]},"public_addr":"172.21.15.1:6803/3758176039","cluster_addr":"172.21.15.1:6805/3758176039","heartbeat_back_addr":"172.21.15.1:6809/3758176039","heartbeat_front_addr":"172.21.15.1:6807/3758176039","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6811","nonce":997975858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6813","nonce":997975858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6817","nonce":997975858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6815","nonce":997975858}]},"public_addr":"172.21.15.1:6811/997975858","cluster_addr":"172.21.15.1:6813/997975858","heartbeat_back_addr":"172.21.15.1:6817/997975858","heartbeat_front_addr":"172.21.15.1:6815/997975858","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6819","nonce":4255524830}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6821","nonce":4255524830}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6825","nonce":4255524830}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6823","nonce":4255524830}]},"public_addr":"172.21.15.1:6819/4255524830","cluster_addr":"172.21.15.1:6821/4255524830","heartbeat_back_addr":"172.21.15.1:6825/4255524830","heartbeat_front_addr":"172.21.15.1:6823/4255524830","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6827","nonce":2316309347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6829","nonce":2316309347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6833","nonce":2316309347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6831","nonce":2316309347}]},"public_addr":"172.21.15.1:6827/2316309347","cluster_addr":"172.21.15.1:6829/2316309347","heartbeat_back_addr":"172.21.15.1:6833/2316309347","heartbeat_front_addr":"172.21.15.1:6831/2316309347","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:41:37.681 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:41:37.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:41:37.930 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[29360]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3661655457' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[40972]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:37 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3661655457' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:37 smithi012 ceph-mon[33640]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:41:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:37 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:37 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3661655457' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:39.853 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:41:39.854 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":67,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T03:41:30.324273+0000","last_up_change":"2024-09-14T03:40:46.998732+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6803","nonce":3758176039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6805","nonce":3758176039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6809","nonce":3758176039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6807","nonce":3758176039}]},"public_addr":"172.21.15.1:6803/3758176039","cluster_addr":"172.21.15.1:6805/3758176039","heartbeat_back_addr":"172.21.15.1:6809/3758176039","heartbeat_front_addr":"172.21.15.1:6807/3758176039","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6811","nonce":997975858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6813","nonce":997975858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6817","nonce":997975858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6815","nonce":997975858}]},"public_addr":"172.21.15.1:6811/997975858","cluster_addr":"172.21.15.1:6813/997975858","heartbeat_back_addr":"172.21.15.1:6817/997975858","heartbeat_front_addr":"172.21.15.1:6815/997975858","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6819","nonce":4255524830}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6821","nonce":4255524830}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6825","nonce":4255524830}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6823","nonce":4255524830}]},"public_addr":"172.21.15.1:6819/4255524830","cluster_addr":"172.21.15.1:6821/4255524830","heartbeat_back_addr":"172.21.15.1:6825/4255524830","heartbeat_front_addr":"172.21.15.1:6823/4255524830","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6827","nonce":2316309347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6829","nonce":2316309347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6833","nonce":2316309347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6831","nonce":2316309347}]},"public_addr":"172.21.15.1:6827/2316309347","cluster_addr":"172.21.15.1:6829/2316309347","heartbeat_back_addr":"172.21.15.1:6833/2316309347","heartbeat_front_addr":"172.21.15.1:6831/2316309347","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:41:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:39 smithi012 ceph-mon[33640]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:40.178 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:39 smithi001 ceph-mon[29360]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:40.178 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:39 smithi001 ceph-mon[40972]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:40.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:41:40.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:41:40.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:41:40.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:41:40.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:41:40.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:41:40.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:41:40.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:41:40.962 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.965 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.966 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.966 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.967 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.967 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:40 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/4203386888' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:40.972 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:40 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/4203386888' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:40.973 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:41.060 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:40 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/4203386888' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:41:42.041 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:41 smithi012 ceph-mon[33640]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:42.045 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:41 smithi001 ceph-mon[29360]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:42.045 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:41 smithi001 ceph-mon[40972]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:42.046 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:41:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:41:41.737Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003634677s 2024-09-14T03:41:42.046 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:41:41] "GET /metrics HTTP/1.1" 200 28487 "" "Prometheus/2.43.0" 2024-09-14T03:41:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:43 smithi012 ceph-mon[33640]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:43.847 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:43 smithi012 podman[65419]: 2024-09-14 03:41:43.823769629 +0000 UTC m=+0.133058853 container create 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.32.0, version=8.5) 2024-09-14T03:41:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:43 smithi001 ceph-mon[29360]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:43.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:43 smithi001 ceph-mon[40972]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:44.269 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 podman[65419]: 2024-09-14 03:41:44.017701459 +0000 UTC m=+0.326990687 container init 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , name=ubi8, release=239.1651231664, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vendor=Red Hat, Inc., distribution-scope=public, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5) 2024-09-14T03:41:44.269 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 podman[65419]: 2024-09-14 03:41:44.023653354 +0000 UTC m=+0.332942579 container start 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., description=Ceph Grafana Container, release=239.1651231664, io.openshift.expose-services=) 2024-09-14T03:41:44.269 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 bash[65419]: 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 2024-09-14T03:41:44.269 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 systemd[1]: Started Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:41:44.270 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186398775Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:41:44.270 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186576687Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:41:44.270 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186596633Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:41:44.270 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186606402Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186613438Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186620156Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186626686Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186633273Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186639784Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186646183Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186652763Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186658983Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186665347Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:41:44.271 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=settings t=2024-09-14T03:41:44.186671723Z level=info msg="App mode production" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore t=2024-09-14T03:41:44.186726411Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore t=2024-09-14T03:41:44.186744043Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.187568901Z level=info msg="Starting DB migrations" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.187645267Z level=info msg="Executing migration" id="create migration_log table" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.188194175Z level=info msg="Executing migration" id="create user table" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.188643799Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.189591194Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.19020206Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.190924736Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.191348013Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.192096123Z level=info msg="Executing migration" id="create user table v2" 2024-09-14T03:41:44.272 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.192526016Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.193360109Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.194225631Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.194944286Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.195898759Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.196513565Z level=info msg="Executing migration" id="Update user table charset" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.197389555Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.198254135Z level=info msg="Executing migration" id="Add missing user data" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.199271328Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.199980037Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.20094952Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.2016654Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.202821503Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.203873092Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-14T03:41:44.273 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.204482955Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.204928581Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.205764669Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.206746472Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.207476497Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.208488466Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.209175451Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.209576411Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.210426865Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.211433062Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.21409942Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.214519878Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.217459814Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-14T03:41:44.274 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.220291639Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.222912236Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.223271646Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.226099727Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.22664291Z level=info msg="Executing migration" id="create star table" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.228958591Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.231738472Z level=info msg="Executing migration" id="create org table v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.234544636Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.237138083Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.239872538Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.242887235Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.245490406Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.248272717Z level=info msg="Executing migration" id="Update org table charset" 2024-09-14T03:41:44.275 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.24858332Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.251457653Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.254229895Z level=info msg="Executing migration" id="create dashboard table" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.256906692Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.257377674Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.260124351Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.262918813Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.265413162Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.265839411Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-14T03:41:44.276 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.268638484Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-14T03:41:44.412 INFO:teuthology.orchestra.run.smithi001.stdout:77309411365 2024-09-14T03:41:44.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:41:44.519 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.271194533Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.273834056Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.274431656Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.277000411Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.279780548Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.282529585Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.283055377Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.285938255Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.288437812Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.289370057Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-14T03:41:44.520 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.290006069Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.291012375Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.291703776Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.29271721Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.293094797Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.293914862Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.294719018Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.295493386Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-14T03:41:44.521 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.296025854Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.296849349Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.297686025Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.298494677Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.298781894Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.299242684Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-14T03:41:44.522 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.299643688Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.300580521Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.301381877Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.301825515Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.302684127Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.30352126Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.304496763Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.305138231Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.305956865Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.30680268Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.307846342Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.308441826Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.309266342Z level=info msg="Executing migration" id="create data_source table" 2024-09-14T03:41:44.523 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.310132088Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.310961209Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.311407096Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.312253068Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.312692138Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.313745293Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.314704203Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.315187948Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.318070926Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.320868358Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.323880398Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.324616163Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.327330093Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-14T03:41:44.524 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.330053685Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.332836496Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.335641441Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.338119599Z level=info msg="Executing migration" id="Add uid column" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.340942884Z level=info msg="Executing migration" id="Update uid value" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.343590478Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.344133874Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.344930294Z level=info msg="Executing migration" id="create api_key table" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.345935511Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.34655854Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.346999455Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.347849374Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.348711764Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-14T03:41:44.525 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.349511556Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.35035052Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.351422729Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.352229944Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.352654701Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.353524879Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.354382363Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.355273912Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.356158371Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.356925692Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.357725193Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.358566214Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.359557514Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.360279166Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-14T03:41:44.526 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.36118895Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.362209932Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.362894642Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.363878893Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.36457995Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.365576251Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.366283735Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.367151068Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.367460818Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.368052646Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.368934253Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.369784673Z level=info msg="Executing migration" id="create quota table v1" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.370774964Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-14T03:41:44.527 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.371487481Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.371766519Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.372175681Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.372594644Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.373167928Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.373469674Z level=info msg="Executing migration" id="create session table" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.373906969Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.374217938Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.374502241Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.374905841Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.375290642Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.37555697Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.375860699Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-14T03:41:44.528 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.376166958Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.376448707Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.37685003Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.377143908Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.377724874Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.3780475Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.378612936Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.379235592Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.379518772Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.379957389Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.380428607Z level=info msg="Executing migration" id="create alert table v1" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.381292612Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.382160054Z level=info msg="Executing migration" id="add index alert state" 2024-09-14T03:41:44.529 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.384894843Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.385328684Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.385717081Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.386150384Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.386566652Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.388001829Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.388388949Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.388833067Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.389160696Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.389501593Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.38990113Z level=info msg="Executing migration" id="Add column is_default" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.390524274Z level=info msg="Executing migration" id="Add column frequency" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.391164198Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-14T03:41:44.530 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.391831693Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.392471106Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.392911035Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.393209296Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.393481469Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.393884713Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.394319364Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.394690294Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.395086497Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.395487526Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.396136531Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.396773991Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.397103334Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-14T03:41:44.531 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.397527197Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.397944739Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.398576449Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.398893362Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.399314294Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.399725389Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.400050504Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.400499165Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.403271043Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.403701352Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-14T03:41:44.532 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.404126673Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-14T03:41:44.533 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.404596765Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-14T03:41:44.533 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.40749155Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.410068967Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.410884525Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.413626125Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.416545048Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.419180902Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.421985912Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.424713931Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.42761749Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.43025915Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.433121859Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-14T03:41:44.536 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.435926938Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.438894992Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.441692196Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.44437909Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.447137284Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.450083769Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.45276824Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.455508964Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.4582906Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.460863239Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.46365322Z level=info msg="Executing migration" id="Move region to single row" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.464254292Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.466791419Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-14T03:41:44.537 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.467267817Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.469755962Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.470247704Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.472111991Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.472587069Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.472967524Z level=info msg="Executing migration" id="create test_data table" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.47551851Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.475987886Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.478746062Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.479264184Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.482069859Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.48263869Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-14T03:41:44.538 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.483001697Z level=info msg="Executing migration" id="create team table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.483795758Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.484315698Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.487269735Z level=info msg="Executing migration" id="create team member table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.489953518Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.492650454Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.493130785Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.496011974Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.498681197Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.501413085Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.502171641Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.505013271Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-14T03:41:44.539 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.50774537Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.510407287Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.510915595Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.513727323Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.51648726Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.519012176Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-14T03:41:44.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.521757488Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-14T03:41:44.770 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.524427341Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-14T03:41:44.771 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.527145552Z level=info msg="Executing migration" id="create tag table" 2024-09-14T03:41:44.771 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.529827573Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-14T03:41:44.771 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.532583914Z level=info msg="Executing migration" id="create login attempt table" 2024-09-14T03:41:44.771 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.535204702Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-14T03:41:44.771 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.537970034Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-14T03:41:44.772 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.540870164Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-14T03:41:44.772 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.543583464Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-14T03:41:44.772 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.546165278Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-14T03:41:44.772 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.548983344Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-14T03:41:44.772 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.549568112Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.550367637Z level=info msg="Executing migration" id="create user auth table" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.55129309Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.552107914Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.552894128Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.55372171Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-14T03:41:44.773 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.554536552Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-14T03:41:44.774 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.555326863Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-14T03:41:44.774 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.556126563Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-14T03:41:44.774 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.556926708Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-14T03:41:44.774 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.557738019Z level=info msg="Executing migration" id="create server_lock table" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.558533345Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.55935776Z level=info msg="Executing migration" id="create user auth token table" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.560179418Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.560978588Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.561967233Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-14T03:41:44.775 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.562600091Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-14T03:41:44.776 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.563438115Z level=info msg="Executing migration" id="create cache_data table" 2024-09-14T03:41:44.776 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.564338769Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-14T03:41:44.776 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.565210871Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-14T03:41:44.776 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.566080493Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.567038254Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.567759988Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.568871391Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.569499969Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.570349603Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.571247708Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.572084793Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.573024238Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.573758368Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.574657343Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-14T03:41:44.777 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.575635363Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.576457877Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.577268863Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.578242414Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.58105034Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.583946526Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.586794269Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.589518054Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.59239332Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.595056185Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.597757051Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.600657207Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.603417391Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-14T03:41:44.778 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.60610798Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-14T03:41:44.779 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.608971268Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-14T03:41:44.779 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.612559852Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.615955547Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.618805862Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.621618611Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.624395471Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.62705113Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.629821341Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.632700633Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.635382069Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.637964061Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.640849281Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.643604839Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-14T03:41:44.780 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.64618745Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.648850572Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.651704347Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.654388373Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.656880247Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.65961378Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.662259522Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.664973966Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.667670609Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.670375993Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.673106341Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.675748702Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-14T03:41:44.781 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.676569897Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.677459011Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.678341831Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.679220441Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.680103085Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.680981103Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.681801871Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.682707147Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.683591016Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.684609196Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.685522047Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.68646327Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.687522167Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-14T03:41:44.782 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.688513542Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.6891682Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.690078201Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.690866017Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.691853122Z level=info msg="Executing migration" id="create alert_image table" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.692537056Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.693548549Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.694300553Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.695289347Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.696003506Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.696097328Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.696866982Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.697843919Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-14T03:41:44.783 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.698486307Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.698613948Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.698746182Z level=info msg="alerts found to migrate" alerts=0 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.699357542Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.700339011Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.701064385Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.701917515Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.702771648Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.703308557Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.704077139Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.704468747Z level=info msg="Executing migration" id="create data_keys table" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.704946206Z level=info msg="Executing migration" id="create secrets table" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.705806392Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-14T03:41:44.784 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.71019937Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.71114191Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.711971824Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.716354444Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.720742193Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.721592954Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.722429584Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.723283227Z level=info msg="Executing migration" id="create permission table" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.724258625Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.725010582Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.725866518Z level=info msg="Executing migration" id="create role table" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.726860379Z level=info msg="Executing migration" id="add column display_name" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.72796234Z level=info msg="Executing migration" id="add column group_name" 2024-09-14T03:41:44.785 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.728907304Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.72977563Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.730699078Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.731667028Z level=info msg="Executing migration" id="create team role table" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.734501751Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.737329961Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.739818299Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.742757874Z level=info msg="Executing migration" id="create user role table" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.745720475Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.748147255Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.751011117Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.753709744Z level=info msg="Executing migration" id="create builtin role table" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.756468501Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-14T03:41:44.786 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.759428107Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.762021902Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.764860142Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.765524191Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.768319275Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.768802382Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.769304297Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.769762716Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.770321589Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-14T03:41:44.787 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.771375368Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.77415557Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.774662786Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.775016783Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.775439991Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.77594743Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.776360527Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.776692021Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.777091091Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.777525531Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.778037452Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-14T03:41:45.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.77906162Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.779405736Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.779883845Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.780359605Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.780846192Z level=info msg="Executing migration" id="add correlation config column" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.781876098Z level=info msg="Executing migration" id="create entity_events table" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.782339862Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.782786841Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.782902051Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:41:45.098 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.783214076Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.78331403Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.785164984Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.785660287Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.787904133Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.791008789Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.793551446Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.796378887Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.799019905Z level=info msg="Executing migration" id="Drop public config table" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.801530576Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.804250606Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.80691982Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:41:45.099 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.809435834Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.812155972Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.815513629Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.818392481Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.820931509Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.823687476Z level=info msg="Executing migration" id="add share column" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.826372531Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.828980805Z level=info msg="Executing migration" id="create file table" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.831746685Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.834404972Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.837163164Z level=info msg="Executing migration" id="create file_meta table" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.837732417Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.83852569Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.8395282Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-14T03:41:45.100 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.84016261Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.841091999Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.841648038Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.842715094Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.843526857Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.844488545Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.845136147Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.846045354Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.84661859Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.84765248Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-14T03:41:45.101 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.848691698Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.849507779Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.859386725Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.860220286Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.861052852Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.864489691Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.865469835Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.866246467Z level=info msg="Executing migration" id="create folder table" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.867086229Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.867967573Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.868912388Z level=info msg="Executing migration" id="Update folder title length" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.869642686Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-14T03:41:45.102 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=migrator t=2024-09-14T03:41:44.870601429Z level=info msg="migrations completed" performed=485 skipped=0 duration=682.972818ms 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore t=2024-09-14T03:41:44.871134738Z level=info msg="Created default organization" 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=plugin.loader t=2024-09-14T03:41:44.932230087Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=plugin.loader t=2024-09-14T03:41:44.961561179Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=plugin.loader t=2024-09-14T03:41:44.961578925Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=secrets t=2024-09-14T03:41:44.961611891Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=query_data t=2024-09-14T03:41:44.963276167Z level=info msg="Query Service initialization" 2024-09-14T03:41:45.103 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=live.push_http t=2024-09-14T03:41:44.965691969Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:41:45.252 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.252 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.252 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.252 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.252 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[29360]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:41:45.253 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.253 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.253 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.253 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.253 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:45 smithi001 ceph-mon[40972]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:41:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-mon[33640]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:41:45.893 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:45 smithi001 systemd[1]: Starting Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:41:46.096 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=infra.usagestats.collector t=2024-09-14T03:41:45.729517573Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:41:46.096 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=provisioning.datasources t=2024-09-14T03:41:45.785195175Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:41:46.096 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=provisioning.datasources t=2024-09-14T03:41:45.817753329Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-14T03:41:46.096 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=provisioning.alerting t=2024-09-14T03:41:45.839959075Z level=info msg="starting to provision alerting" 2024-09-14T03:41:46.096 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=provisioning.alerting t=2024-09-14T03:41:45.839979883Z level=info msg="finished to provision alerting" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=grafanaStorageLogger t=2024-09-14T03:41:45.84007344Z level=info msg="storage starting" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=ngalert.state.manager t=2024-09-14T03:41:45.84033668Z level=info msg="Warming state cache for startup" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=http.server t=2024-09-14T03:41:45.844240406Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore.transactions t=2024-09-14T03:41:45.85054344Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=ngalert.state.manager t=2024-09-14T03:41:45.850666122Z level=info msg="State cache has been initialized" states=0 duration=10.327284ms 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=ticker t=2024-09-14T03:41:45.850750531Z level=info msg=starting first_tick=2024-09-14T03:41:50Z 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:41:45.850747974Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore.transactions t=2024-09-14T03:41:45.896542342Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore.transactions t=2024-09-14T03:41:45.937606483Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:41:46.097 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:41:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[65433]: logger=sqlstore.transactions t=2024-09-14T03:41:45.948050035Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-14T03:41:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:46 smithi001 ceph-mon[29360]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:46 smithi001 ceph-mon[40972]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:46 smithi012 ceph-mon[33640]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:47.369 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:47.370 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:47.370 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:47.370 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:41:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:41:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:48 smithi001 ceph-mon[29360]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:48 smithi001 ceph-mon[40972]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:48 smithi012 ceph-mon[33640]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:50.210 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:50 smithi001 ceph-mon[29360]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:50.210 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:50 smithi001 ceph-mon[40972]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:50 smithi012 ceph-mon[33640]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:51.969 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:41:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:41:51] "GET /metrics HTTP/1.1" 200 28486 "" "Prometheus/2.43.0" 2024-09-14T03:41:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:52 smithi001 ceph-mon[29360]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:52 smithi001 ceph-mon[40972]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:52 smithi012 ceph-mon[33640]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:54 smithi001 ceph-mon[29360]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:54 smithi001 ceph-mon[40972]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:54.439 INFO:teuthology.orchestra.run.smithi001.stdout:55834574892 2024-09-14T03:41:54.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:41:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:54 smithi012 ceph-mon[33640]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:54.890 INFO:teuthology.orchestra.run.smithi001.stdout:133143986203 2024-09-14T03:41:54.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:41:55.312 INFO:teuthology.orchestra.run.smithi001.stdout:201863462933 2024-09-14T03:41:55.313 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:41:55.660 INFO:teuthology.orchestra.run.smithi001.stdout:167503724568 2024-09-14T03:41:55.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:41:55.675 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:56.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:56 smithi001 ceph-mon[40972]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:56 smithi001 ceph-mon[29360]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:56 smithi012 ceph-mon[33640]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:56.597 INFO:teuthology.orchestra.run.smithi001.stdout:34359738416 2024-09-14T03:41:56.597 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:41:56.840 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:56 smithi001 bash[74650]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:41:57.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:57 smithi001 bash[74650]: Getting image source signatures 2024-09-14T03:41:57.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:57 smithi001 bash[74650]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:41:57.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:57 smithi001 bash[74650]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:41:57.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:57 smithi001 bash[74650]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:41:57.417 INFO:teuthology.orchestra.run.smithi001.stdout:111669149730 2024-09-14T03:41:57.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:41:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:41:58 smithi001 ceph-mon[29360]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:41:58 smithi001 ceph-mon[40972]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:41:58 smithi012 ceph-mon[33640]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:41:59.194 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:59.195 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:59.197 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:59.211 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:58 smithi001 bash[74650]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:41:59.470 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:59 smithi001 bash[74650]: Writing manifest to image destination 2024-09-14T03:41:59.939 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:59.939 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:59.939 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:00.130 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:41:59 smithi001 podman[74650]: 2024-09-14 03:41:59.747031232 +0000 UTC m=+4.086946670 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:42:00.130 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 podman[74650]: 2024-09-14 03:42:00.008580498 +0000 UTC m=+4.348495917 container create d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-mon[29360]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-mon[40972]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:00.472 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 podman[74650]: 2024-09-14 03:42:00.354632032 +0000 UTC m=+4.694547459 container init d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:00.472 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 podman[74650]: 2024-09-14 03:42:00.360389852 +0000 UTC m=+4.700305271 container start d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:00.472 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.364Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:42:00.472 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.364Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.364Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:42:00.473 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.365Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:42:00.474 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:42:00.475 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:42:00.476 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:42:00.477 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:42:00.478 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:42:00.478 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[75217]: ts=2024-09-14T03:42:00.366Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:42:00.478 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 bash[74650]: d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba 2024-09-14T03:42:00.478 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:42:00 smithi001 systemd[1]: Started Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:00 smithi012 ceph-mon[33640]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:00.779 INFO:teuthology.orchestra.run.smithi001.stdout:236223201296 2024-09-14T03:42:00.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:42:01.594 INFO:teuthology.orchestra.run.smithi001.stdout:77309411368 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[29360]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.596 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-mon[40972]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:42:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:01 smithi012 ceph-mon[33640]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:42:02.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:01] "GET /metrics HTTP/1.1" 200 28485 "" "Prometheus/2.43.0" 2024-09-14T03:42:02.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:01 smithi012 bash[65714]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:42:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[29360]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3148543474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:42:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[40972]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3148543474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:42:02.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:02 smithi012 ceph-mon[33640]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:02 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3148543474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:42:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:03.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:03 smithi012 bash[65714]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:42:03.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:03 smithi012 bash[65714]: Writing manifest to image destination 2024-09-14T03:42:03.994 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:03 smithi012 podman[65714]: 2024-09-14 03:42:03.610281088 +0000 UTC m=+2.264855115 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:42:03.995 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:03 smithi012 podman[65714]: 2024-09-14 03:42:03.708804453 +0000 UTC m=+2.363378513 container create 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:42:04.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 podman[65714]: 2024-09-14 03:42:04.094452074 +0000 UTC m=+2.749026088 container init 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:42:04.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 podman[65714]: 2024-09-14 03:42:04.100190232 +0000 UTC m=+2.754764257 container start 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:42:04.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.104Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:42:04.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.104Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:42:04.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.105Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.105Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.105Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.105Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:42:04.348 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:42:04.349 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:42:04.350 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:42:04.351 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:42:04.352 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:42:04.353 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:42:04.353 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:42:04.354 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[65790]: ts=2024-09-14T03:42:04.106Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:42:04.354 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 bash[65714]: 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c 2024-09-14T03:42:04.354 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:42:04 smithi012 systemd[1]: Started Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.586 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:04 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:04 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:05.663 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411368 for osd.2 2024-09-14T03:42:05.664 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:06 smithi012 ceph-mon[33640]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:06.920 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:06 smithi001 ceph-mon[29360]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:06.920 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:06 smithi001 ceph-mon[40972]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:07.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:07 smithi001 ceph-mon[40972]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:07.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:07 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:07.894 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:07 smithi001 ceph-mon[29360]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:07.894 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:07 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:08.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:07 smithi012 ceph-mon[33640]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:08.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:07 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:08.622 INFO:teuthology.orchestra.run.smithi001.stdout:167503724571 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2032560218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.110 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:08 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2032560218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:42:09.137 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724568 got 167503724571 for osd.5 2024-09-14T03:42:09.137 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:08 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:08 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:08 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2032560218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:42:09.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:09 smithi001 ceph-mon[29360]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:09 smithi001 ceph-mon[40972]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:09.990 INFO:teuthology.orchestra.run.smithi001.stdout:201863462936 2024-09-14T03:42:10.191 INFO:teuthology.orchestra.run.smithi001.stdout:55834574895 2024-09-14T03:42:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:09 smithi012 ceph-mon[33640]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:11.044 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:10 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1983928708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:42:11.045 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:10 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2933605010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:42:11.045 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:10 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1983928708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:42:11.045 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:10 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2933605010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:42:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:10 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1983928708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:42:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:10 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2933605010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:42:11.362 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574895 for osd.1 2024-09-14T03:42:11.363 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:11.616 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462933 got 201863462936 for osd.6 2024-09-14T03:42:11.616 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:11.684 INFO:teuthology.orchestra.run.smithi001.stdout:133143986208 2024-09-14T03:42:12.192 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:11 smithi001 ceph-mon[40972]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:12.192 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:11 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/303870011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:42:12.193 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:11 smithi001 ceph-mon[29360]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:12.193 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:11 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/303870011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:42:12.193 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:11] "GET /metrics HTTP/1.1" 200 28485 "" "Prometheus/2.43.0" 2024-09-14T03:42:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:11 smithi012 ceph-mon[33640]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:11 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/303870011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:42:12.460 INFO:teuthology.orchestra.run.smithi001.stdout:34359738420 2024-09-14T03:42:12.776 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986208 for osd.4 2024-09-14T03:42:12.777 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:13.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:12 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1810097311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:42:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:12 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1810097311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:42:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:12 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1810097311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:42:13.440 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:14.106 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:13 smithi001 ceph-mon[29360]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:14.106 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:13 smithi001 ceph-mon[40972]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:13 smithi012 ceph-mon[33640]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:14.610 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738420 for osd.0 2024-09-14T03:42:14.610 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:15.047 INFO:teuthology.orchestra.run.smithi001.stdout:111669149735 2024-09-14T03:42:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:14 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2607613382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:42:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:14 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2607613382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:42:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:14 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2607613382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:42:15.651 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149735 for osd.3 2024-09-14T03:42:15.651 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:16.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:15 smithi001 ceph-mon[29360]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:16.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:15 smithi001 ceph-mon[40972]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:15 smithi012 ceph-mon[33640]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:16.491 INFO:teuthology.orchestra.run.smithi001.stdout:236223201299 2024-09-14T03:42:17.153 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:16 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2365055777' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:42:17.153 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:17.154 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:16 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2365055777' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:42:17.154 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:17.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:16 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2365055777' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:42:17.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:17.528 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201296 got 236223201299 for osd.7 2024-09-14T03:42:17.528 DEBUG:teuthology.parallel:result is None 2024-09-14T03:42:17.529 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:42:17.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:42:17.759 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:18.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:17 smithi001 ceph-mon[29360]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:18.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:17 smithi001 ceph-mon[40972]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:17 smithi012 ceph-mon[33640]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:20.107 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:19 smithi001 ceph-mon[29360]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:20.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:19 smithi001 ceph-mon[40972]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:19 smithi012 ceph-mon[33640]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:20.535 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:20.535 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T03:42:21.367 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-09-14T03:42:18.678158+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":239592,"kb_used_data":7048,"kb_used_omap":12,"kb_used_meta":232435,"kb_avail":749721624,"statfs":{"total":767960285184,"available":767714942976,"internally_reserved":0,"allocated":7217152,"data_stored":4057445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":238014008},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002300"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774661+0000","last_change":"2024-09-14T03:41:23.740297+0000","last_active":"2024-09-14T03:41:30.774661+0000","last_peered":"2024-09-14T03:41:30.774661+0000","last_clean":"2024-09-14T03:41:30.774661+0000","last_became_active":"2024-09-14T03:41:18.713102+0000","last_became_peered":"2024-09-14T03:41:18.713102+0000","last_unstale":"2024-09-14T03:41:30.774661+0000","last_undegraded":"2024-09-14T03:41:30.774661+0000","last_fullsized":"2024-09-14T03:41:30.774661+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:37:04.215608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773677+0000","last_change":"2024-09-14T03:41:23.736591+0000","last_active":"2024-09-14T03:41:30.773677+0000","last_peered":"2024-09-14T03:41:30.773677+0000","last_clean":"2024-09-14T03:41:30.773677+0000","last_became_active":"2024-09-14T03:41:19.397811+0000","last_became_peered":"2024-09-14T03:41:19.397811+0000","last_unstale":"2024-09-14T03:41:30.773677+0000","last_undegraded":"2024-09-14T03:41:30.773677+0000","last_fullsized":"2024-09-14T03:41:30.773677+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:41.233137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040775400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774590+0000","last_change":"2024-09-14T03:41:23.741514+0000","last_active":"2024-09-14T03:41:30.774590+0000","last_peered":"2024-09-14T03:41:30.774590+0000","last_clean":"2024-09-14T03:41:30.774590+0000","last_became_active":"2024-09-14T03:41:19.397559+0000","last_became_peered":"2024-09-14T03:41:19.397559+0000","last_unstale":"2024-09-14T03:41:30.774590+0000","last_undegraded":"2024-09-14T03:41:30.774590+0000","last_fullsized":"2024-09-14T03:41:30.774590+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:33:59.613404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773653+0000","last_change":"2024-09-14T03:41:23.736609+0000","last_active":"2024-09-14T03:41:30.773653+0000","last_peered":"2024-09-14T03:41:30.773653+0000","last_clean":"2024-09-14T03:41:30.773653+0000","last_became_active":"2024-09-14T03:41:18.713287+0000","last_became_peered":"2024-09-14T03:41:18.713287+0000","last_unstale":"2024-09-14T03:41:30.773653+0000","last_undegraded":"2024-09-14T03:41:30.773653+0000","last_fullsized":"2024-09-14T03:41:30.773653+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:09.426148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774297+0000","last_change":"2024-09-14T03:41:23.740892+0000","last_active":"2024-09-14T03:41:30.774297+0000","last_peered":"2024-09-14T03:41:30.774297+0000","last_clean":"2024-09-14T03:41:30.774297+0000","last_became_active":"2024-09-14T03:41:18.713385+0000","last_became_peered":"2024-09-14T03:41:18.713385+0000","last_unstale":"2024-09-14T03:41:30.774297+0000","last_undegraded":"2024-09-14T03:41:30.774297+0000","last_fullsized":"2024-09-14T03:41:30.774297+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:58.288379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773963+0000","last_change":"2024-09-14T03:41:23.737002+0000","last_active":"2024-09-14T03:41:30.773963+0000","last_peered":"2024-09-14T03:41:30.773963+0000","last_clean":"2024-09-14T03:41:30.773963+0000","last_became_active":"2024-09-14T03:41:18.715814+0000","last_became_peered":"2024-09-14T03:41:18.715814+0000","last_unstale":"2024-09-14T03:41:30.773963+0000","last_undegraded":"2024-09-14T03:41:30.773963+0000","last_fullsized":"2024-09-14T03:41:30.773963+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:04.446799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774404+0000","last_change":"2024-09-14T03:41:23.741581+0000","last_active":"2024-09-14T03:41:30.774404+0000","last_peered":"2024-09-14T03:41:30.774404+0000","last_clean":"2024-09-14T03:41:30.774404+0000","last_became_active":"2024-09-14T03:41:18.714655+0000","last_became_peered":"2024-09-14T03:41:18.714655+0000","last_unstale":"2024-09-14T03:41:30.774404+0000","last_undegraded":"2024-09-14T03:41:30.774404+0000","last_fullsized":"2024-09-14T03:41:30.774404+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:56.242065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774344+0000","last_change":"2024-09-14T03:41:23.740334+0000","last_active":"2024-09-14T03:41:30.774344+0000","last_peered":"2024-09-14T03:41:30.774344+0000","last_clean":"2024-09-14T03:41:30.774344+0000","last_became_active":"2024-09-14T03:41:18.712934+0000","last_became_peered":"2024-09-14T03:41:18.712934+0000","last_unstale":"2024-09-14T03:41:30.774344+0000","last_undegraded":"2024-09-14T03:41:30.774344+0000","last_fullsized":"2024-09-14T03:41:30.774344+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:30:42.245712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":262,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-14T03:41:23.741010+0000","last_change":"2024-09-14T03:40:51.032125+0000","last_active":"2024-09-14T03:41:23.741010+0000","last_peered":"2024-09-14T03:41:23.741010+0000","last_clean":"2024-09-14T03:41:23.741010+0000","last_became_active":"2024-09-14T03:40:51.031753+0000","last_became_peered":"2024-09-14T03:40:51.031753+0000","last_unstale":"2024-09-14T03:41:23.741010+0000","last_undegraded":"2024-09-14T03:41:23.741010+0000","last_fullsized":"2024-09-14T03:41:23.741010+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:30:14.411793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52600000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.50900000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.375},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.495}]},{"osd":5,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.441}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32356,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712796,"statfs":{"total":95995035648,"available":95961903104,"internally_reserved":0,"allocated":1269760,"data_stored":875757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.49299999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.70199999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56000000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35099999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.621},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.54400000000000004,"15min":0.53400000000000003},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56799999999999995,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:42:06 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.39900000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.40600000000000003,"15min":0.39900000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60699999999999998,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:41:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.60699999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67100000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52700000000000002},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.72399999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.63800000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.74299999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.45500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55700000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71999999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56799999999999995,"15min":0.56499999999999995},"min":{"1min":0.46500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41999999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.77000000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.40200000000000002,"15min":0.39600000000000002},"min":{"1min":0.32200000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.438},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:41:51 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45300000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31780,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713372,"statfs":{"total":95995035648,"available":95962492928,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.26600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.42599999999999999,"15min":0.42799999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.63500000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.438,"15min":0.44},"min":{"1min":0.33400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.65400000000000003,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45500000000000002,"15min":0.45300000000000001},"min":{"1min":0.30399999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.57899999999999996}]},{"osd":2,"last update":"Sat Sep 14 03:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.48099999999999998,"15min":0.48499999999999999},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.42399999999999999,"15min":0.42599999999999999},"min":{"1min":0.309,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.41999999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80200000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.495}]},{"osd":6,"last update":"Sat Sep 14 03:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:41:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.70599999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31780,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713372,"statfs":{"total":95995035648,"available":95962492928,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38},"min":{"1min":0.26800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.308},{"interface":"front","average":{"1min":0.371,"5min":0.377,"15min":0.371},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.501,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.39900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39500000000000002,"15min":0.40100000000000002},"min":{"1min":0.31,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:41:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.43099999999999999,"15min":0.42699999999999999},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.38},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.45000000000000001,"15min":0.45400000000000001},"min":{"1min":0.317,"5min":0.31,"15min":0.31},"max":{"1min":0.54700000000000004,"5min":0.629,"15min":0.629},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71599999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.623,"5min":0.57699999999999996,"15min":0.56899999999999995},"min":{"1min":0.51900000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82599999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.47599999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.432},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.495}]},{"osd":6,"last update":"Sat Sep 14 03:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:41:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58199999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27688,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717464,"statfs":{"total":95995035648,"available":95966683136,"internally_reserved":0,"allocated":684032,"data_stored":285943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"min":{"1min":0.191,"5min":0.185,"15min":0.185},"max":{"1min":0.55500000000000005,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.377,"5min":0.34599999999999997,"15min":0.32300000000000001},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.64100000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.308}]},{"osd":2,"last update":"Sat Sep 14 03:41:53 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.49299999999999999,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.26700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.60599999999999998,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71699999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.56000000000000005,"15min":0.56999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.64600000000000002,"15min":0.65600000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.871,"15min":0.871},"last":0.74299999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.72699999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:41:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.69299999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32360,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712792,"statfs":{"total":95995035648,"available":95961899008,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.35199999999999998,"15min":0.33000000000000002},"min":{"1min":0.28699999999999998,"5min":0.224,"15min":0.224},"max":{"1min":0.51400000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.36399999999999999,"15min":0.32900000000000001},"min":{"1min":0.30299999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.624,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.28599999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.45700000000000002,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.37},"min":{"1min":0.28699999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.47399999999999998,"15min":0.47699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.54000000000000004,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48799999999999999,"15min":0.48999999999999999},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.621,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.59299999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.624,"15min":0.629},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.88600000000000001,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.78700000000000003,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.68000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:42:21.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:42:21.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:21.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:21.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:21 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:21.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:21] "GET /metrics HTTP/1.1" 200 28485 "" "Prometheus/2.43.0" 2024-09-14T03:42:22.745 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:22 smithi012 ceph-mon[33640]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:22 smithi012 ceph-mon[33640]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:42:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:22 smithi012 ceph-mon[33640]: Reconfiguring daemon alertmanager.a on smithi001 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[29360]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[29360]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[29360]: Reconfiguring daemon alertmanager.a on smithi001 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[40972]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[40972]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:42:22.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:22 smithi001 ceph-mon[40972]: Reconfiguring daemon alertmanager.a on smithi001 2024-09-14T03:42:24.327 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:24 smithi001 systemd[1]: Stopping Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:42:24.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:24 smithi001 ceph-mon[29360]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:24.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:24 smithi001 ceph-mon[40972]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:25.014 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:24 smithi012 ceph-mon[33640]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:25.687 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:25 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[72136]: ts=2024-09-14T03:42:25.388Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:42:25.687 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:25 smithi001 podman[77555]: 2024-09-14 03:42:25.40965552 +0000 UTC m=+0.136038019 container died ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:26.362 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:26.362 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T03:42:26.363 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:26 smithi001 podman[77555]: 2024-09-14 03:42:26.005542923 +0000 UTC m=+0.731925416 container cleanup ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:26.363 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:26 smithi001 bash[77555]: ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:42:26.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:26 smithi001 ceph-mon[40972]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:26.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:26 smithi001 ceph-mon[29360]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:26.721 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:26 smithi001 podman[77587]: 2024-09-14 03:42:26.466433975 +0000 UTC m=+1.058143552 container remove ae6d8347dd5779afa8058d28720cb3fa7fc3ae8503f68ebea816656242b20a22 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:26.721 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:26 smithi001 podman[77587]: 2024-09-14 03:42:26.550358348 +0000 UTC m=+1.142067881 volume remove 81d2a5c88bd4e0ae46621fba4c32c137b427c2411a2e22e0fddfdee53472d98c 2024-09-14T03:42:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:26 smithi012 ceph-mon[33640]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:27.218 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":44,"stamp":"2024-09-14T03:42:24.679279+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":239592,"kb_used_data":7048,"kb_used_omap":12,"kb_used_meta":232435,"kb_avail":749721624,"statfs":{"total":767960285184,"available":767714942976,"internally_reserved":0,"allocated":7217152,"data_stored":4057445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":238014008},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002153"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774661+0000","last_change":"2024-09-14T03:41:23.740297+0000","last_active":"2024-09-14T03:41:30.774661+0000","last_peered":"2024-09-14T03:41:30.774661+0000","last_clean":"2024-09-14T03:41:30.774661+0000","last_became_active":"2024-09-14T03:41:18.713102+0000","last_became_peered":"2024-09-14T03:41:18.713102+0000","last_unstale":"2024-09-14T03:41:30.774661+0000","last_undegraded":"2024-09-14T03:41:30.774661+0000","last_fullsized":"2024-09-14T03:41:30.774661+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:37:04.215608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773677+0000","last_change":"2024-09-14T03:41:23.736591+0000","last_active":"2024-09-14T03:41:30.773677+0000","last_peered":"2024-09-14T03:41:30.773677+0000","last_clean":"2024-09-14T03:41:30.773677+0000","last_became_active":"2024-09-14T03:41:19.397811+0000","last_became_peered":"2024-09-14T03:41:19.397811+0000","last_unstale":"2024-09-14T03:41:30.773677+0000","last_undegraded":"2024-09-14T03:41:30.773677+0000","last_fullsized":"2024-09-14T03:41:30.773677+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:41.233137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040775400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774590+0000","last_change":"2024-09-14T03:41:23.741514+0000","last_active":"2024-09-14T03:41:30.774590+0000","last_peered":"2024-09-14T03:41:30.774590+0000","last_clean":"2024-09-14T03:41:30.774590+0000","last_became_active":"2024-09-14T03:41:19.397559+0000","last_became_peered":"2024-09-14T03:41:19.397559+0000","last_unstale":"2024-09-14T03:41:30.774590+0000","last_undegraded":"2024-09-14T03:41:30.774590+0000","last_fullsized":"2024-09-14T03:41:30.774590+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:33:59.613404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773653+0000","last_change":"2024-09-14T03:41:23.736609+0000","last_active":"2024-09-14T03:41:30.773653+0000","last_peered":"2024-09-14T03:41:30.773653+0000","last_clean":"2024-09-14T03:41:30.773653+0000","last_became_active":"2024-09-14T03:41:18.713287+0000","last_became_peered":"2024-09-14T03:41:18.713287+0000","last_unstale":"2024-09-14T03:41:30.773653+0000","last_undegraded":"2024-09-14T03:41:30.773653+0000","last_fullsized":"2024-09-14T03:41:30.773653+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:09.426148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774297+0000","last_change":"2024-09-14T03:41:23.740892+0000","last_active":"2024-09-14T03:41:30.774297+0000","last_peered":"2024-09-14T03:41:30.774297+0000","last_clean":"2024-09-14T03:41:30.774297+0000","last_became_active":"2024-09-14T03:41:18.713385+0000","last_became_peered":"2024-09-14T03:41:18.713385+0000","last_unstale":"2024-09-14T03:41:30.774297+0000","last_undegraded":"2024-09-14T03:41:30.774297+0000","last_fullsized":"2024-09-14T03:41:30.774297+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:58.288379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.773963+0000","last_change":"2024-09-14T03:41:23.737002+0000","last_active":"2024-09-14T03:41:30.773963+0000","last_peered":"2024-09-14T03:41:30.773963+0000","last_clean":"2024-09-14T03:41:30.773963+0000","last_became_active":"2024-09-14T03:41:18.715814+0000","last_became_peered":"2024-09-14T03:41:18.715814+0000","last_unstale":"2024-09-14T03:41:30.773963+0000","last_undegraded":"2024-09-14T03:41:30.773963+0000","last_fullsized":"2024-09-14T03:41:30.773963+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:04.446799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774404+0000","last_change":"2024-09-14T03:41:23.741581+0000","last_active":"2024-09-14T03:41:30.774404+0000","last_peered":"2024-09-14T03:41:30.774404+0000","last_clean":"2024-09-14T03:41:30.774404+0000","last_became_active":"2024-09-14T03:41:18.714655+0000","last_became_peered":"2024-09-14T03:41:18.714655+0000","last_unstale":"2024-09-14T03:41:30.774404+0000","last_undegraded":"2024-09-14T03:41:30.774404+0000","last_fullsized":"2024-09-14T03:41:30.774404+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:56.242065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:41:30.774344+0000","last_change":"2024-09-14T03:41:23.740334+0000","last_active":"2024-09-14T03:41:30.774344+0000","last_peered":"2024-09-14T03:41:30.774344+0000","last_clean":"2024-09-14T03:41:30.774344+0000","last_became_active":"2024-09-14T03:41:18.712934+0000","last_became_peered":"2024-09-14T03:41:18.712934+0000","last_unstale":"2024-09-14T03:41:30.774344+0000","last_undegraded":"2024-09-14T03:41:30.774344+0000","last_fullsized":"2024-09-14T03:41:30.774344+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:30:42.245712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":262,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-14T03:41:23.741010+0000","last_change":"2024-09-14T03:40:51.032125+0000","last_active":"2024-09-14T03:41:23.741010+0000","last_peered":"2024-09-14T03:41:23.741010+0000","last_clean":"2024-09-14T03:41:23.741010+0000","last_became_active":"2024-09-14T03:40:51.031753+0000","last_became_peered":"2024-09-14T03:40:51.031753+0000","last_unstale":"2024-09-14T03:41:23.741010+0000","last_undegraded":"2024-09-14T03:41:23.741010+0000","last_fullsized":"2024-09-14T03:41:23.741010+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:30:14.411793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.5}]},{"osd":1,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.44}]},{"osd":4,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.371},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.56299999999999994}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32356,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712796,"statfs":{"total":95995035648,"available":95961903104,"internally_reserved":0,"allocated":1269760,"data_stored":875757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.61899999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.63100000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56000000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35099999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.628},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.54400000000000004,"15min":0.53400000000000003},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.61299999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.78100000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:42:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56799999999999995,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.69199999999999995}]},{"osd":4,"last update":"Sat Sep 14 03:42:06 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.39900000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.40600000000000003,"15min":0.39900000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60699999999999998,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.65700000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:41:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.76200000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67100000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52700000000000002},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.72399999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.74299999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.45500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55700000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71999999999999997,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54000000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:41:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56799999999999995,"15min":0.56499999999999995},"min":{"1min":0.46500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41999999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.77000000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.434}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.40200000000000002,"15min":0.39600000000000002},"min":{"1min":0.32200000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.73399999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:41:51 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51500000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31780,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713372,"statfs":{"total":95995035648,"available":95962492928,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.39800000000000002,"15min":0.40100000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.20200000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.55000000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:42:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.26200000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.55700000000000005,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60399999999999998,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:42:18 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.47099999999999997,"15min":0.48099999999999998},"min":{"1min":0.29499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.58199999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.41799999999999998,"15min":0.42399999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.58699999999999997,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.24099999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.373},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.41999999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80200000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:41:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.41399999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31780,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713372,"statfs":{"total":95995035648,"available":95962492928,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38},"min":{"1min":0.26800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.377,"15min":0.371},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.501,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.312}]},{"osd":1,"last update":"Sat Sep 14 03:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39500000000000002,"15min":0.40100000000000002},"min":{"1min":0.31,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:42:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.51600000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.437,"15min":0.45000000000000001},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":0.52400000000000002,"5min":0.629,"15min":0.629},"last":0.376}]},{"osd":4,"last update":"Sat Sep 14 03:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71599999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.755},{"interface":"front","average":{"1min":0.623,"5min":0.57699999999999996,"15min":0.56899999999999995},"min":{"1min":0.51900000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82599999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.47599999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70299999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.71099999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:41:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.63800000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27688,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717464,"statfs":{"total":95995035648,"available":95966683136,"internally_reserved":0,"allocated":684032,"data_stored":285943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"min":{"1min":0.191,"5min":0.185,"15min":0.185},"max":{"1min":0.55500000000000005,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.377,"5min":0.34599999999999997,"15min":0.32300000000000001},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.64100000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:41:53 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.49299999999999999,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:42:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.46000000000000002,"15min":0.46200000000000002},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71699999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.56000000000000005,"15min":0.56999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.624}]},{"osd":5,"last update":"Sat Sep 14 03:42:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.64600000000000002,"15min":0.65600000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.871,"15min":0.871},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:41:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.66600000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32360,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712792,"statfs":{"total":95995035648,"available":95961899008,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.35199999999999998,"15min":0.33000000000000002},"min":{"1min":0.28699999999999998,"5min":0.224,"15min":0.224},"max":{"1min":0.51400000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.36399999999999999,"15min":0.32900000000000001},"min":{"1min":0.30299999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.624,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.28599999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.45700000000000002,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.37},"min":{"1min":0.28699999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.47399999999999998,"15min":0.47699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.54000000000000004,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48799999999999999,"15min":0.48999999999999999},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.621,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.59299999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:42:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.624,"15min":0.629},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.88600000000000001,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.78700000000000003,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:41:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.68000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:42:27.220 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:42:27.220 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:42:27.221 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:42:27.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:42:27.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:42:27.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 systemd[1]: Stopped Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:27.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 systemd[1]: Starting Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:42:27.857 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:27 smithi001 ceph-mon[29360]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:27.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:27 smithi001 ceph-mon[40972]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:27.857 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 podman[77799]: 2024-09-14 03:42:27.419046227 +0000 UTC m=+0.017613069 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:42:27.857 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 podman[77799]: 2024-09-14 03:42:27.538685638 +0000 UTC m=+0.137252468 volume create 88ab7845ea8682dd5160b388b7a68f1fa7e7713c66efc0f68196723cc713c09a 2024-09-14T03:42:27.857 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:27 smithi001 podman[77799]: 2024-09-14 03:42:27.630600614 +0000 UTC m=+0.229167442 container create 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:27 smithi012 ceph-mon[33640]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:42:28.181 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 podman[77799]: 2024-09-14 03:42:28.031034979 +0000 UTC m=+0.629601807 container init 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 podman[77799]: 2024-09-14 03:42:28.036295132 +0000 UTC m=+0.634861970 container start 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.049Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.049Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.051Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.1 port=9094 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.052Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.075Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.075Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.078Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:42:28.286 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:28.078Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:42:28.287 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 bash[77799]: 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 2024-09-14T03:42:28.287 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:28 smithi001 systemd[1]: Started Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:28.890 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[29360]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:28.890 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:28.890 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:28.891 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[40972]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:28.891 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:28.891 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:28 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:28.900 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:28 smithi012 ceph-mon[33640]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:28.900 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:28 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:28.900 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:28 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:29 smithi001 ceph-mon[29360]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:42:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:29 smithi001 ceph-mon[29360]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:42:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:29 smithi001 ceph-mon[40972]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:42:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:29 smithi001 ceph-mon[40972]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:42:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:29 smithi012 ceph-mon[33640]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:42:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:29 smithi012 ceph-mon[33640]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:42:29.876 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:29.876 INFO:teuthology.orchestra.run.smithi001.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:42:30.198 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:30.052Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000685467s 2024-09-14T03:42:30.501 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:42:30.501 INFO:tasks.cephadm:Setup complete, yielding 2024-09-14T03:42:30.501 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-14T03:42:30.514 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi001.front.sepia.ceph.com 2024-09-14T03:42:30.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-14T03:42:30.728 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:30.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:30 smithi001 ceph-mon[40972]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:30.849 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:30 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3060931243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:42:30.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:30 smithi001 ceph-mon[29360]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:30.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:30 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3060931243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:42:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:30 smithi012 ceph-mon[33640]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:30 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3060931243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:42:31.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 systemd[1]: Stopping Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:42:32.161 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:31] "GET /metrics HTTP/1.1" 200 28484 "" "Prometheus/2.43.0" 2024-09-14T03:42:32.228 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:42:32.228 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[63332]: ts=2024-09-14T03:42:31.945Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:42:32.229 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:31 smithi012 podman[66668]: 2024-09-14 03:42:31.955636364 +0000 UTC m=+0.138924593 container died 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:32.586 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 podman[66668]: 2024-09-14 03:42:32.325730808 +0000 UTC m=+0.509019025 container cleanup 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:32.586 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 bash[66668]: ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:42:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:32 smithi012 ceph-mon[33640]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:32.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 podman[66685]: 2024-09-14 03:42:32.585202004 +0000 UTC m=+0.627636016 container remove 2d782d2745e866cf38aa5932a5f467654893af172042ff34e6c9b7bbf13173ef (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:32.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:42:32.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 systemd[1]: Stopped Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:32.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.181s CPU time. 2024-09-14T03:42:32.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 systemd[1]: Starting Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:42:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:32 smithi001 ceph-mon[29360]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:32.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:32.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:32 smithi001 ceph-mon[40972]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:32.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:33.239 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:32 smithi012 podman[66782]: 2024-09-14 03:42:32.87354477 +0000 UTC m=+0.017419591 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:42:33.239 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 podman[66782]: 2024-09-14 03:42:33.036144435 +0000 UTC m=+0.180019250 container create 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:33.239 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 podman[66782]: 2024-09-14 03:42:33.238153771 +0000 UTC m=+0.382028584 container init 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:33.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 podman[66782]: 2024-09-14 03:42:33.243164399 +0000 UTC m=+0.387039214 container start 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.267Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.268Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.268Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi012 (none))" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.268Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.268Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.269Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.270Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.270Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.270Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.273Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:42:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.273Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.608µs 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.273Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.274Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.274Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.274Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.228µs wal_replay_duration=1.346881ms wbl_replay_duration=111ns total_replay_duration=1.385901ms 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.275Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.275Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.275Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.297Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.544842ms db_storage=688ns remote_storage=1.092µs web_handler=377ns query_engine=1.042µs scrape=378.726µs scrape_sd=95.477µs notify=17.775µs notify_sd=28.693µs rules=20.706818ms tracing=5.481µs 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.297Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:42:33.297Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 bash[66782]: 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 2024-09-14T03:42:33.598 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:42:33 smithi012 systemd[1]: Started Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:42:33.663 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:42:33.663 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.663 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:33.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STOPPING 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STOPPED 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STARTING 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Serving on http://:::9283 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STARTED 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STOPPING 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STOPPED 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STARTING 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Serving on http://:::9283 2024-09-14T03:42:33.666 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STARTED 2024-09-14T03:42:33.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:33] ENGINE Bus STOPPING 2024-09-14T03:42:34.066 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:42:34.066 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:34.067 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:33 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:34.470 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:42:34.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:34] ENGINE Bus STOPPED 2024-09-14T03:42:34.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:34] ENGINE Bus STARTING 2024-09-14T03:42:34.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:34] ENGINE Serving on http://:::9283 2024-09-14T03:42:34.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:42:34] ENGINE Bus STARTED 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:42:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:42:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:34 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi001.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:42:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:42:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:42:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:34 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[29337]: 2024-09-14T03:42:34.634+0000 7f858ea7f640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:42:35.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-14T03:42:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:42:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:42:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: fsmap foo:0 2024-09-14T03:42:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:35.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:42:35.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:42:35.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:35.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:42:35.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:42:35.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:42:35.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:42:35.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: fsmap foo:0 2024-09-14T03:42:35.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:35 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: fsmap foo:0 2024-09-14T03:42:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:35 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:36.297 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[29360]: Saving service mds.foo spec with placement count:2 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[29360]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[29360]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[40972]: Saving service mds.foo spec with placement count:2 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[40972]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:36.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:36 smithi001 ceph-mon[40972]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:42:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:36 smithi012 ceph-mon[33640]: Saving service mds.foo spec with placement count:2 2024-09-14T03:42:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:36 smithi012 ceph-mon[33640]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:36 smithi012 ceph-mon[33640]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:42:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[29360]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:42:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[40972]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:42:37.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:37.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:37 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:37 smithi012 ceph-mon[33640]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:42:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:37 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:37 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.319 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:42:38.054Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002221195s 2024-09-14T03:42:38.938 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[29360]: Deploying daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:38 smithi001 ceph-mon[40972]: Deploying daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:42:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 234 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:39.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:38 smithi012 ceph-mon[33640]: Deploying daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:42:39.862 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:39 smithi001 ceph-mon[40972]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:39.862 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:39 smithi001 ceph-mon[29360]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:39.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-14T03:42:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:39 smithi012 ceph-mon[33640]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:40.556 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:41.851 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:41 smithi001 ceph-mon[29360]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:41.851 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:41 smithi001 ceph-mon[40972]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:42.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:41 smithi012 ceph-mon[33640]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:42.130 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:41] "GET /metrics HTTP/1.1" 200 28484 "" "Prometheus/2.43.0" 2024-09-14T03:42:43.109 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T03:42:43.118 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:42:43.118 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:42:43.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:42:43.421 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: Deploying daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:boot 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: daemon mds.foo.smithi001.ytjgbj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: Cluster is now healthy 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: fsmap foo:0 1 up:standby 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:creating} 2024-09-14T03:42:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[40972]: daemon mds.foo.smithi001.ytjgbj is now active in filesystem foo as rank 0 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: Deploying daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:boot 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: daemon mds.foo.smithi001.ytjgbj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:42:44.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: Cluster is now healthy 2024-09-14T03:42:44.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: fsmap foo:0 1 up:standby 2024-09-14T03:42:44.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:42:44.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:creating} 2024-09-14T03:42:44.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:44 smithi001 ceph-mon[29360]: daemon mds.foo.smithi001.ytjgbj is now active in filesystem foo as rank 0 2024-09-14T03:42:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: Deploying daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:boot 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: daemon mds.foo.smithi001.ytjgbj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: fsmap foo:0 1 up:standby 2024-09-14T03:42:44.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:42:44.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:creating} 2024-09-14T03:42:44.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:44 smithi012 ceph-mon[33640]: daemon mds.foo.smithi001.ytjgbj is now active in filesystem foo as rank 0 2024-09-14T03:42:45.461 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:45 smithi012 ceph-mon[33640]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:45 smithi012 ceph-mon[33640]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:45 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 2024-09-14T03:42:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:45 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2694791643' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:42:46.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[29360]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:46.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[29360]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:46.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 2024-09-14T03:42:46.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2694791643' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:42:46.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[40972]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:46.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[40972]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:46.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 2024-09-14T03:42:46.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:45 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2694791643' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:42:46.163 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":20,"flags":0,"active_gid":24350,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6800","nonce":2630182969},{"type":"v1","addr":"172.21.15.1:6801","nonce":2630182969}]},"active_addr":"172.21.15.1:6801/2630182969","active_change":"2024-09-14T03:41:16.654863+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":739412744}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":416113000}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":3674234575}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":2817796360}]}]} 2024-09-14T03:42:46.167 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:42:46.167 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:42:46.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:42:46.382 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:47.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:boot 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:boot 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:42:47.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:46 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:boot 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:42:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:46 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:48.211 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:47 smithi001 ceph-mon[29360]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:48.211 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:47 smithi001 ceph-mon[40972]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:48.307 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:47 smithi012 ceph-mon[33640]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:42:48.923 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:48.923 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":72,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T03:42:36.652333+0000","last_up_change":"2024-09-14T03:40:46.998732+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6803","nonce":3758176039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6805","nonce":3758176039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6809","nonce":3758176039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6807","nonce":3758176039}]},"public_addr":"172.21.15.1:6803/3758176039","cluster_addr":"172.21.15.1:6805/3758176039","heartbeat_back_addr":"172.21.15.1:6809/3758176039","heartbeat_front_addr":"172.21.15.1:6807/3758176039","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6811","nonce":997975858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6813","nonce":997975858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6817","nonce":997975858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6815","nonce":997975858}]},"public_addr":"172.21.15.1:6811/997975858","cluster_addr":"172.21.15.1:6813/997975858","heartbeat_back_addr":"172.21.15.1:6817/997975858","heartbeat_front_addr":"172.21.15.1:6815/997975858","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6819","nonce":4255524830}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6821","nonce":4255524830}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6825","nonce":4255524830}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6823","nonce":4255524830}]},"public_addr":"172.21.15.1:6819/4255524830","cluster_addr":"172.21.15.1:6821/4255524830","heartbeat_back_addr":"172.21.15.1:6825/4255524830","heartbeat_front_addr":"172.21.15.1:6823/4255524830","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6827","nonce":2316309347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6829","nonce":2316309347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6833","nonce":2316309347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6831","nonce":2316309347}]},"public_addr":"172.21.15.1:6827/2316309347","cluster_addr":"172.21.15.1:6829/2316309347","heartbeat_back_addr":"172.21.15.1:6833/2316309347","heartbeat_front_addr":"172.21.15.1:6831/2316309347","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[29360]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/338908622' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[40972]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:48 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/338908622' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:48 smithi012 ceph-mon[33640]: mds.? [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] up:active 2024-09-14T03:42:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:48 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:48 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/338908622' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:49.678 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:42:49.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:42:50.462 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:50.745 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[29360]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:50.745 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:50.745 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:50.746 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[40972]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:50.746 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:50.746 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:50 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:50 smithi012 ceph-mon[33640]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:50 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:50 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:51 smithi001 ceph-mon[29360]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:standby 2024-09-14T03:42:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:51 smithi001 ceph-mon[29360]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:51 smithi001 ceph-mon[40972]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:standby 2024-09-14T03:42:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:51 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:51 smithi012 ceph-mon[33640]: mds.? [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] up:standby 2024-09-14T03:42:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:51 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:42:52.152 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:42:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:42:51] "GET /metrics HTTP/1.1" 200 33391 "" "Prometheus/2.43.0" 2024-09-14T03:42:52.520 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:42:52.521 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":72,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T03:42:36.652333+0000","last_up_change":"2024-09-14T03:40:46.998732+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6803","nonce":3758176039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6805","nonce":3758176039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6809","nonce":3758176039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3758176039},{"type":"v1","addr":"172.21.15.1:6807","nonce":3758176039}]},"public_addr":"172.21.15.1:6803/3758176039","cluster_addr":"172.21.15.1:6805/3758176039","heartbeat_back_addr":"172.21.15.1:6809/3758176039","heartbeat_front_addr":"172.21.15.1:6807/3758176039","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6811","nonce":997975858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6813","nonce":997975858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6817","nonce":997975858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":997975858},{"type":"v1","addr":"172.21.15.1:6815","nonce":997975858}]},"public_addr":"172.21.15.1:6811/997975858","cluster_addr":"172.21.15.1:6813/997975858","heartbeat_back_addr":"172.21.15.1:6817/997975858","heartbeat_front_addr":"172.21.15.1:6815/997975858","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6819","nonce":4255524830}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6821","nonce":4255524830}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6825","nonce":4255524830}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4255524830},{"type":"v1","addr":"172.21.15.1:6823","nonce":4255524830}]},"public_addr":"172.21.15.1:6819/4255524830","cluster_addr":"172.21.15.1:6821/4255524830","heartbeat_back_addr":"172.21.15.1:6825/4255524830","heartbeat_front_addr":"172.21.15.1:6823/4255524830","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6827","nonce":2316309347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6829","nonce":2316309347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6833","nonce":2316309347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":2316309347},{"type":"v1","addr":"172.21.15.1:6831","nonce":2316309347}]},"public_addr":"172.21.15.1:6827/2316309347","cluster_addr":"172.21.15.1:6829/2316309347","heartbeat_back_addr":"172.21.15.1:6833/2316309347","heartbeat_front_addr":"172.21.15.1:6831/2316309347","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:42:52.759 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:52 smithi012 ceph-mon[33640]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:52.861 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:52 smithi001 ceph-mon[29360]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:52.861 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:52 smithi001 ceph-mon[40972]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:53.552 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3120952556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:53.552 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.552 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.552 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3120952556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.553 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:53 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:53.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:42:53.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:42:53.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:42:53.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:42:53.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:42:53.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:42:53.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:42:53.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3120952556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:53.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:53 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:54.615 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:54.643 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:54.656 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:54.660 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:54 smithi001 ceph-mon[29360]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:54 smithi001 ceph-mon[40972]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:54 smithi012 ceph-mon[33640]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:56.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[40972]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:56.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:56.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:56.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[29360]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:56.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:56.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:56 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:56 smithi012 ceph-mon[33640]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:56 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:56 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:42:57.536 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:42:58 smithi001 ceph-mon[29360]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:58.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:42:58 smithi001 ceph-mon[40972]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:42:58 smithi012 ceph-mon[33640]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:42:59.625 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:42:59.625 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:00.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:00 smithi001 ceph-mon[29360]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:43:00.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:00 smithi001 ceph-mon[40972]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:43:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:00 smithi012 ceph-mon[33640]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:43:01.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:01] "GET /metrics HTTP/1.1" 200 33681 "" "Prometheus/2.43.0" 2024-09-14T03:43:02.071 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:02.470 INFO:teuthology.orchestra.run.smithi001.stdout:34359738431 2024-09-14T03:43:02.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:43:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:02 smithi001 ceph-mon[29360]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:02 smithi001 ceph-mon[40972]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:02 smithi012 ceph-mon[33640]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:04.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:04 smithi001 ceph-mon[29360]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:04.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:04 smithi001 ceph-mon[40972]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:04 smithi012 ceph-mon[33640]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:43:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:06 smithi012 ceph-mon[33640]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:06 smithi001 ceph-mon[29360]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:06 smithi001 ceph-mon[40972]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:07.010 INFO:teuthology.orchestra.run.smithi001.stdout:111669149745 2024-09-14T03:43:07.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:43:07.020 INFO:teuthology.orchestra.run.smithi001.stdout:77309411381 2024-09-14T03:43:07.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:43:07.768 INFO:teuthology.orchestra.run.smithi001.stdout:55834574907 2024-09-14T03:43:07.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:43:08.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:08 smithi001 ceph-mon[29360]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:08.763 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:08 smithi001 ceph-mon[40972]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:08 smithi012 ceph-mon[33640]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:09.832 INFO:teuthology.orchestra.run.smithi001.stdout:236223201311 2024-09-14T03:43:09.833 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:43:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:10 smithi012 ceph-mon[33640]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:10 smithi001 ceph-mon[29360]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:10 smithi001 ceph-mon[40972]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:11.429 INFO:teuthology.orchestra.run.smithi001.stdout:167503724585 2024-09-14T03:43:11.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:43:11.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:11] "GET /metrics HTTP/1.1" 200 33681 "" "Prometheus/2.43.0" 2024-09-14T03:43:12.445 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:12.446 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:12.446 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:12.477 INFO:teuthology.orchestra.run.smithi001.stdout:201863462948 2024-09-14T03:43:12.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:43:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:12.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:12.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:12.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:12.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:12.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:12 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:12 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:13.447 INFO:teuthology.orchestra.run.smithi001.stdout:133143986222 2024-09-14T03:43:13.447 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:43:14.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:14 smithi001 ceph-mon[29360]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:14.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:14 smithi001 ceph-mon[40972]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:14 smithi012 ceph-mon[33640]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:15.345 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:15.370 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:15.374 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:16.781 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:16 smithi001 ceph-mon[29360]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:16.782 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:16 smithi001 ceph-mon[40972]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:16 smithi012 ceph-mon[33640]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:17.024 INFO:teuthology.orchestra.run.smithi001.stdout:236223201312 2024-09-14T03:43:17.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-14T03:43:17.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3249022457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:17.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:17 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3249022457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:43:17.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201312 for osd.7 2024-09-14T03:43:17.729 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-14T03:43:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-14T03:43:17.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:17.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:17 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3249022457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[29360]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[29360]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[40972]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-14T03:43:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-14T03:43:18.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-14T03:43:18.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:18 smithi001 ceph-mon[40972]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:43:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:18 smithi012 ceph-mon[33640]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:18 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-14T03:43:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:18 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-14T03:43:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:18 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-14T03:43:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:18 smithi012 ceph-mon[33640]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:43:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:19 smithi001 ceph-mon[29360]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:43:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:19 smithi001 ceph-mon[40972]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:43:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:19 smithi012 ceph-mon[33640]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:43:20.045 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:20.046 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:20.695 INFO:teuthology.orchestra.run.smithi001.stdout:55834574910 2024-09-14T03:43:20.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:20 smithi001 ceph-mon[29360]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:20.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:20 smithi001 ceph-mon[40972]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:20 smithi012 ceph-mon[33640]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:21.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:21 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3811887422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:43:21.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:21 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3811887422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:43:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:21 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3811887422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:43:22.192 INFO:teuthology.orchestra.run.smithi001.stdout:201863462951 2024-09-14T03:43:22.192 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:21] "GET /metrics HTTP/1.1" 200 33680 "" "Prometheus/2.43.0" 2024-09-14T03:43:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:22 smithi001 ceph-mon[29360]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:22 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1669939593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:43:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:22 smithi001 ceph-mon[40972]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:22 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1669939593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:43:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:22 smithi012 ceph-mon[33640]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:22 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1669939593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:43:24.297 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574910 for osd.1 2024-09-14T03:43:24.297 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:24.818 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:24 smithi001 ceph-mon[29360]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:24.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:24 smithi001 ceph-mon[40972]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:24 smithi012 ceph-mon[33640]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:25.202 INFO:teuthology.orchestra.run.smithi001.stdout:34359738436 2024-09-14T03:43:25.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:25 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/620461589' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:43:25.820 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:25 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/620461589' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:43:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:25 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/620461589' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:43:25.955 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462951 for osd.6 2024-09-14T03:43:25.956 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:26.300 INFO:teuthology.orchestra.run.smithi001.stdout:111669149750 2024-09-14T03:43:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:26 smithi012 ceph-mon[33640]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:26 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/701348897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:43:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:26 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2885876999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[29360]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/701348897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2885876999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[40972]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/701348897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:43:26.952 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:26 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2885876999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:43:27.803 INFO:teuthology.orchestra.run.smithi001.stdout:133143986224 2024-09-14T03:43:28.590 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986224 for osd.4 2024-09-14T03:43:28.590 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:28 smithi001 ceph-mon[29360]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:28 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2914897524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:43:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:28 smithi001 ceph-mon[40972]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:28 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2914897524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:43:28.736 INFO:teuthology.orchestra.run.smithi001.stdout:167503724588 2024-09-14T03:43:28.788 INFO:teuthology.orchestra.run.smithi001.stdout:77309411386 2024-09-14T03:43:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:28 smithi012 ceph-mon[33640]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:28 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2914897524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:43:29.594 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411381 got 77309411386 for osd.2 2024-09-14T03:43:29.594 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:29 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/4266010579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:43:29.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:29 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/4266010579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:43:29.959 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:29 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/4266010579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:43:29.988 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724588 for osd.5 2024-09-14T03:43:29.988 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:30.464 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738431 got 34359738436 for osd.0 2024-09-14T03:43:30.464 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:30 smithi001 ceph-mon[29360]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:30 smithi001 ceph-mon[40972]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:30 smithi012 ceph-mon[33640]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:30.910 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149745 got 111669149750 for osd.3 2024-09-14T03:43:30.911 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:30.911 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:43:30.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:43:31.126 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:32.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:31] "GET /metrics HTTP/1.1" 200 33692 "" "Prometheus/2.43.0" 2024-09-14T03:43:32.652 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:32 smithi001 ceph-mon[29360]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:32.653 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:32.653 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:32 smithi001 ceph-mon[40972]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:32.653 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:32 smithi012 ceph-mon[33640]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:33.080 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:43:33.081 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T03:43:33.705 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-09-14T03:43:32.693321+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":240380,"kb_used_data":7816,"kb_used_omap":12,"kb_used_meta":232435,"kb_avail":749720836,"statfs":{"total":767960285184,"available":767714136064,"internally_reserved":0,"allocated":8003584,"data_stored":4468619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":238014008},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002287"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634992+0000","last_change":"2024-09-14T03:43:18.634992+0000","last_active":"2024-09-14T03:43:18.634992+0000","last_peered":"2024-09-14T03:43:18.634992+0000","last_clean":"2024-09-14T03:43:18.634992+0000","last_became_active":"2024-09-14T03:43:18.634473+0000","last_became_peered":"2024-09-14T03:43:18.634473+0000","last_unstale":"2024-09-14T03:43:18.634992+0000","last_undegraded":"2024-09-14T03:43:18.634992+0000","last_fullsized":"2024-09-14T03:43:18.634992+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:32.160322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551646+0000","last_change":"2024-09-14T03:42:34.647856+0000","last_active":"2024-09-14T03:43:17.551646+0000","last_peered":"2024-09-14T03:43:17.551646+0000","last_clean":"2024-09-14T03:43:17.551646+0000","last_became_active":"2024-09-14T03:42:34.647566+0000","last_became_peered":"2024-09-14T03:42:34.647566+0000","last_unstale":"2024-09-14T03:43:17.551646+0000","last_undegraded":"2024-09-14T03:43:17.551646+0000","last_fullsized":"2024-09-14T03:43:17.551646+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:35.220436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548652+0000","last_change":"2024-09-14T03:42:34.882870+0000","last_active":"2024-09-14T03:43:17.548652+0000","last_peered":"2024-09-14T03:43:17.548652+0000","last_clean":"2024-09-14T03:43:17.548652+0000","last_became_active":"2024-09-14T03:42:34.882555+0000","last_became_peered":"2024-09-14T03:42:34.882555+0000","last_unstale":"2024-09-14T03:43:17.548652+0000","last_undegraded":"2024-09-14T03:43:17.548652+0000","last_fullsized":"2024-09-14T03:43:17.548652+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:34.325899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148346+0000","last_change":"2024-09-14T03:42:34.649795+0000","last_active":"2024-09-14T03:43:18.148346+0000","last_peered":"2024-09-14T03:43:18.148346+0000","last_clean":"2024-09-14T03:43:18.148346+0000","last_became_active":"2024-09-14T03:42:34.649531+0000","last_became_peered":"2024-09-14T03:42:34.649531+0000","last_unstale":"2024-09-14T03:43:18.148346+0000","last_undegraded":"2024-09-14T03:43:18.148346+0000","last_fullsized":"2024-09-14T03:43:18.148346+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:01.527597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148589+0000","last_change":"2024-09-14T03:42:35.664546+0000","last_active":"2024-09-14T03:43:18.148589+0000","last_peered":"2024-09-14T03:43:18.148589+0000","last_clean":"2024-09-14T03:43:18.148589+0000","last_became_active":"2024-09-14T03:42:35.664287+0000","last_became_peered":"2024-09-14T03:42:35.664287+0000","last_unstale":"2024-09-14T03:43:18.148589+0000","last_undegraded":"2024-09-14T03:43:18.148589+0000","last_fullsized":"2024-09-14T03:43:18.148589+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:11.241309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551248+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T03:43:17.551248+0000","last_peered":"2024-09-14T03:43:17.551248+0000","last_clean":"2024-09-14T03:43:17.551248+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T03:43:17.551248+0000","last_undegraded":"2024-09-14T03:43:17.551248+0000","last_fullsized":"2024-09-14T03:43:17.551248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549689+0000","last_change":"2024-09-14T03:42:35.665434+0000","last_active":"2024-09-14T03:43:17.549689+0000","last_peered":"2024-09-14T03:43:17.549689+0000","last_clean":"2024-09-14T03:43:17.549689+0000","last_became_active":"2024-09-14T03:42:35.665037+0000","last_became_peered":"2024-09-14T03:42:35.665037+0000","last_unstale":"2024-09-14T03:43:17.549689+0000","last_undegraded":"2024-09-14T03:43:17.549689+0000","last_fullsized":"2024-09-14T03:43:17.549689+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:06.613986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550233+0000","last_change":"2024-09-14T03:42:34.644354+0000","last_active":"2024-09-14T03:43:17.550233+0000","last_peered":"2024-09-14T03:43:17.550233+0000","last_clean":"2024-09-14T03:43:17.550233+0000","last_became_active":"2024-09-14T03:42:34.644093+0000","last_became_peered":"2024-09-14T03:42:34.644093+0000","last_unstale":"2024-09-14T03:43:17.550233+0000","last_undegraded":"2024-09-14T03:43:17.550233+0000","last_fullsized":"2024-09-14T03:43:17.550233+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:07.595587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551942+0000","last_change":"2024-09-14T03:42:34.648183+0000","last_active":"2024-09-14T03:43:17.551942+0000","last_peered":"2024-09-14T03:43:17.551942+0000","last_clean":"2024-09-14T03:43:17.551942+0000","last_became_active":"2024-09-14T03:42:34.647786+0000","last_became_peered":"2024-09-14T03:42:34.647786+0000","last_unstale":"2024-09-14T03:43:17.551942+0000","last_undegraded":"2024-09-14T03:43:17.551942+0000","last_fullsized":"2024-09-14T03:43:17.551942+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:30:45.555570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548255+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T03:43:17.548255+0000","last_peered":"2024-09-14T03:43:17.548255+0000","last_clean":"2024-09-14T03:43:17.548255+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T03:43:17.548255+0000","last_undegraded":"2024-09-14T03:43:17.548255+0000","last_fullsized":"2024-09-14T03:43:17.548255+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551883+0000","last_change":"2024-09-14T03:42:34.643514+0000","last_active":"2024-09-14T03:43:17.551883+0000","last_peered":"2024-09-14T03:43:17.551883+0000","last_clean":"2024-09-14T03:43:17.551883+0000","last_became_active":"2024-09-14T03:42:34.643235+0000","last_became_peered":"2024-09-14T03:42:34.643235+0000","last_unstale":"2024-09-14T03:43:17.551883+0000","last_undegraded":"2024-09-14T03:43:17.551883+0000","last_fullsized":"2024-09-14T03:43:17.551883+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:26.173830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547137+0000","last_change":"2024-09-14T03:42:35.664330+0000","last_active":"2024-09-14T03:43:17.547137+0000","last_peered":"2024-09-14T03:43:17.547137+0000","last_clean":"2024-09-14T03:43:17.547137+0000","last_became_active":"2024-09-14T03:42:35.664023+0000","last_became_peered":"2024-09-14T03:42:35.664023+0000","last_unstale":"2024-09-14T03:43:17.547137+0000","last_undegraded":"2024-09-14T03:43:17.547137+0000","last_fullsized":"2024-09-14T03:43:17.547137+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:30.956305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551763+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T03:43:17.551763+0000","last_peered":"2024-09-14T03:43:17.551763+0000","last_clean":"2024-09-14T03:43:17.551763+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T03:43:17.551763+0000","last_undegraded":"2024-09-14T03:43:17.551763+0000","last_fullsized":"2024-09-14T03:43:17.551763+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550269+0000","last_change":"2024-09-14T03:42:35.664374+0000","last_active":"2024-09-14T03:43:17.550269+0000","last_peered":"2024-09-14T03:43:17.550269+0000","last_clean":"2024-09-14T03:43:17.550269+0000","last_became_active":"2024-09-14T03:42:35.664099+0000","last_became_peered":"2024-09-14T03:42:35.664099+0000","last_unstale":"2024-09-14T03:43:17.550269+0000","last_undegraded":"2024-09-14T03:43:17.550269+0000","last_fullsized":"2024-09-14T03:43:17.550269+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:31.081575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548825+0000","last_change":"2024-09-14T03:42:34.641776+0000","last_active":"2024-09-14T03:43:17.548825+0000","last_peered":"2024-09-14T03:43:17.548825+0000","last_clean":"2024-09-14T03:43:17.548825+0000","last_became_active":"2024-09-14T03:42:34.641470+0000","last_became_peered":"2024-09-14T03:42:34.641470+0000","last_unstale":"2024-09-14T03:43:17.548825+0000","last_undegraded":"2024-09-14T03:43:17.548825+0000","last_fullsized":"2024-09-14T03:43:17.548825+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.940502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148406+0000","last_change":"2024-09-14T03:42:35.666256+0000","last_active":"2024-09-14T03:43:18.148406+0000","last_peered":"2024-09-14T03:43:18.148406+0000","last_clean":"2024-09-14T03:43:18.148406+0000","last_became_active":"2024-09-14T03:42:35.665990+0000","last_became_peered":"2024-09-14T03:42:35.665990+0000","last_unstale":"2024-09-14T03:43:18.148406+0000","last_undegraded":"2024-09-14T03:43:18.148406+0000","last_fullsized":"2024-09-14T03:43:18.148406+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:37.777147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549149+0000","last_change":"2024-09-14T03:42:34.643600+0000","last_active":"2024-09-14T03:43:17.549149+0000","last_peered":"2024-09-14T03:43:17.549149+0000","last_clean":"2024-09-14T03:43:17.549149+0000","last_became_active":"2024-09-14T03:42:34.643303+0000","last_became_peered":"2024-09-14T03:42:34.643303+0000","last_unstale":"2024-09-14T03:43:17.549149+0000","last_undegraded":"2024-09-14T03:43:17.549149+0000","last_fullsized":"2024-09-14T03:43:17.549149+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:20.221506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547317+0000","last_change":"2024-09-14T03:42:35.661473+0000","last_active":"2024-09-14T03:43:17.547317+0000","last_peered":"2024-09-14T03:43:17.547317+0000","last_clean":"2024-09-14T03:43:17.547317+0000","last_became_active":"2024-09-14T03:42:35.661188+0000","last_became_peered":"2024-09-14T03:42:35.661188+0000","last_unstale":"2024-09-14T03:43:17.547317+0000","last_undegraded":"2024-09-14T03:43:17.547317+0000","last_fullsized":"2024-09-14T03:43:17.547317+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:51:30.698952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550482+0000","last_change":"2024-09-14T03:42:34.646942+0000","last_active":"2024-09-14T03:43:17.550482+0000","last_peered":"2024-09-14T03:43:17.550482+0000","last_clean":"2024-09-14T03:43:17.550482+0000","last_became_active":"2024-09-14T03:42:34.646614+0000","last_became_peered":"2024-09-14T03:42:34.646614+0000","last_unstale":"2024-09-14T03:43:17.550482+0000","last_undegraded":"2024-09-14T03:43:17.550482+0000","last_fullsized":"2024-09-14T03:43:17.550482+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:21:15.407821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547841+0000","last_change":"2024-09-14T03:42:35.664990+0000","last_active":"2024-09-14T03:43:17.547841+0000","last_peered":"2024-09-14T03:43:17.547841+0000","last_clean":"2024-09-14T03:43:17.547841+0000","last_became_active":"2024-09-14T03:42:35.664644+0000","last_became_peered":"2024-09-14T03:42:35.664644+0000","last_unstale":"2024-09-14T03:43:17.547841+0000","last_undegraded":"2024-09-14T03:43:17.547841+0000","last_fullsized":"2024-09-14T03:43:17.547841+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:58.814241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547857+0000","last_change":"2024-09-14T03:42:34.650432+0000","last_active":"2024-09-14T03:43:17.547857+0000","last_peered":"2024-09-14T03:43:17.547857+0000","last_clean":"2024-09-14T03:43:17.547857+0000","last_became_active":"2024-09-14T03:42:34.650103+0000","last_became_peered":"2024-09-14T03:42:34.650103+0000","last_unstale":"2024-09-14T03:43:17.547857+0000","last_undegraded":"2024-09-14T03:43:17.547857+0000","last_fullsized":"2024-09-14T03:43:17.547857+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:12.350046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547756+0000","last_change":"2024-09-14T03:42:35.664372+0000","last_active":"2024-09-14T03:43:17.547756+0000","last_peered":"2024-09-14T03:43:17.547756+0000","last_clean":"2024-09-14T03:43:17.547756+0000","last_became_active":"2024-09-14T03:42:35.664063+0000","last_became_peered":"2024-09-14T03:42:35.664063+0000","last_unstale":"2024-09-14T03:43:17.547756+0000","last_undegraded":"2024-09-14T03:43:17.547756+0000","last_fullsized":"2024-09-14T03:43:17.547756+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:12:19.444102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548535+0000","last_change":"2024-09-14T03:42:34.643342+0000","last_active":"2024-09-14T03:43:17.548535+0000","last_peered":"2024-09-14T03:43:17.548535+0000","last_clean":"2024-09-14T03:43:17.548535+0000","last_became_active":"2024-09-14T03:42:34.643059+0000","last_became_peered":"2024-09-14T03:42:34.643059+0000","last_unstale":"2024-09-14T03:43:17.548535+0000","last_undegraded":"2024-09-14T03:43:17.548535+0000","last_fullsized":"2024-09-14T03:43:17.548535+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:29.317772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546136+0000","last_change":"2024-09-14T03:42:35.667459+0000","last_active":"2024-09-14T03:43:17.546136+0000","last_peered":"2024-09-14T03:43:17.546136+0000","last_clean":"2024-09-14T03:43:17.546136+0000","last_became_active":"2024-09-14T03:42:35.667192+0000","last_became_peered":"2024-09-14T03:42:35.667192+0000","last_unstale":"2024-09-14T03:43:17.546136+0000","last_undegraded":"2024-09-14T03:43:17.546136+0000","last_fullsized":"2024-09-14T03:43:17.546136+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:31.239739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":270,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551520+0000","last_change":"2024-09-14T03:40:51.032125+0000","last_active":"2024-09-14T03:43:17.551520+0000","last_peered":"2024-09-14T03:43:17.551520+0000","last_clean":"2024-09-14T03:43:17.551520+0000","last_became_active":"2024-09-14T03:40:51.031753+0000","last_became_peered":"2024-09-14T03:40:51.031753+0000","last_unstale":"2024-09-14T03:43:17.551520+0000","last_undegraded":"2024-09-14T03:43:17.551520+0000","last_fullsized":"2024-09-14T03:43:17.551520+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:30:14.411793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.550021+0000","last_change":"2024-09-14T03:41:23.740334+0000","last_active":"2024-09-14T03:43:18.550021+0000","last_peered":"2024-09-14T03:43:18.550021+0000","last_clean":"2024-09-14T03:43:18.550021+0000","last_became_active":"2024-09-14T03:41:18.712934+0000","last_became_peered":"2024-09-14T03:41:18.712934+0000","last_unstale":"2024-09-14T03:43:18.550021+0000","last_undegraded":"2024-09-14T03:43:18.550021+0000","last_fullsized":"2024-09-14T03:43:18.550021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:30:42.245712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548966+0000","last_change":"2024-09-14T03:42:34.640923+0000","last_active":"2024-09-14T03:43:17.548966+0000","last_peered":"2024-09-14T03:43:17.548966+0000","last_clean":"2024-09-14T03:43:17.548966+0000","last_became_active":"2024-09-14T03:42:34.640629+0000","last_became_peered":"2024-09-14T03:42:34.640629+0000","last_unstale":"2024-09-14T03:43:17.548966+0000","last_undegraded":"2024-09-14T03:43:17.548966+0000","last_fullsized":"2024-09-14T03:43:17.548966+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:45.483399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550716+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T03:43:17.550716+0000","last_peered":"2024-09-14T03:43:17.550716+0000","last_clean":"2024-09-14T03:43:17.550716+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T03:43:17.550716+0000","last_undegraded":"2024-09-14T03:43:17.550716+0000","last_fullsized":"2024-09-14T03:43:17.550716+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633310+0000","last_change":"2024-09-14T03:41:23.741581+0000","last_active":"2024-09-14T03:43:18.633310+0000","last_peered":"2024-09-14T03:43:18.633310+0000","last_clean":"2024-09-14T03:43:18.633310+0000","last_became_active":"2024-09-14T03:41:18.714655+0000","last_became_peered":"2024-09-14T03:41:18.714655+0000","last_unstale":"2024-09-14T03:43:18.633310+0000","last_undegraded":"2024-09-14T03:43:18.633310+0000","last_fullsized":"2024-09-14T03:43:18.633310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:56.242065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148205+0000","last_change":"2024-09-14T03:42:34.648700+0000","last_active":"2024-09-14T03:43:18.148205+0000","last_peered":"2024-09-14T03:43:18.148205+0000","last_clean":"2024-09-14T03:43:18.148205+0000","last_became_active":"2024-09-14T03:42:34.648392+0000","last_became_peered":"2024-09-14T03:42:34.648392+0000","last_unstale":"2024-09-14T03:43:18.148205+0000","last_undegraded":"2024-09-14T03:43:18.148205+0000","last_fullsized":"2024-09-14T03:43:18.148205+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:21.595422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148547+0000","last_change":"2024-09-14T03:42:35.669177+0000","last_active":"2024-09-14T03:43:18.148547+0000","last_peered":"2024-09-14T03:43:18.148547+0000","last_clean":"2024-09-14T03:43:18.148547+0000","last_became_active":"2024-09-14T03:42:35.668885+0000","last_became_peered":"2024-09-14T03:42:35.668885+0000","last_unstale":"2024-09-14T03:43:18.148547+0000","last_undegraded":"2024-09-14T03:43:18.148547+0000","last_fullsized":"2024-09-14T03:43:18.148547+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:08.424889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633512+0000","last_change":"2024-09-14T03:41:23.737002+0000","last_active":"2024-09-14T03:43:18.633512+0000","last_peered":"2024-09-14T03:43:18.633512+0000","last_clean":"2024-09-14T03:43:18.633512+0000","last_became_active":"2024-09-14T03:41:18.715814+0000","last_became_peered":"2024-09-14T03:41:18.715814+0000","last_unstale":"2024-09-14T03:43:18.633512+0000","last_undegraded":"2024-09-14T03:43:18.633512+0000","last_fullsized":"2024-09-14T03:43:18.633512+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:04.446799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.544922+0000","last_change":"2024-09-14T03:42:34.636512+0000","last_active":"2024-09-14T03:43:17.544922+0000","last_peered":"2024-09-14T03:43:17.544922+0000","last_clean":"2024-09-14T03:43:17.544922+0000","last_became_active":"2024-09-14T03:42:34.636117+0000","last_became_peered":"2024-09-14T03:42:34.636117+0000","last_unstale":"2024-09-14T03:43:17.544922+0000","last_undegraded":"2024-09-14T03:43:17.544922+0000","last_fullsized":"2024-09-14T03:43:17.544922+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:58.012169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546217+0000","last_change":"2024-09-14T03:42:35.827893+0000","last_active":"2024-09-14T03:43:17.546217+0000","last_peered":"2024-09-14T03:43:17.546217+0000","last_clean":"2024-09-14T03:43:17.546217+0000","last_became_active":"2024-09-14T03:42:35.827559+0000","last_became_peered":"2024-09-14T03:42:35.827559+0000","last_unstale":"2024-09-14T03:43:17.546217+0000","last_undegraded":"2024-09-14T03:43:17.546217+0000","last_fullsized":"2024-09-14T03:43:17.546217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:29:31.099286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":30,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.549849+0000","last_change":"2024-09-14T03:41:23.740892+0000","last_active":"2024-09-14T03:43:18.549849+0000","last_peered":"2024-09-14T03:43:18.549849+0000","last_clean":"2024-09-14T03:43:18.549849+0000","last_became_active":"2024-09-14T03:41:18.713385+0000","last_became_peered":"2024-09-14T03:41:18.713385+0000","last_unstale":"2024-09-14T03:43:18.549849+0000","last_undegraded":"2024-09-14T03:43:18.549849+0000","last_fullsized":"2024-09-14T03:43:18.549849+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:58.288379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548492+0000","last_change":"2024-09-14T03:42:34.650038+0000","last_active":"2024-09-14T03:43:17.548492+0000","last_peered":"2024-09-14T03:43:17.548492+0000","last_clean":"2024-09-14T03:43:17.548492+0000","last_became_active":"2024-09-14T03:42:34.649777+0000","last_became_peered":"2024-09-14T03:42:34.649777+0000","last_unstale":"2024-09-14T03:43:17.548492+0000","last_undegraded":"2024-09-14T03:43:17.548492+0000","last_fullsized":"2024-09-14T03:43:17.548492+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:03:45.718863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634920+0000","last_change":"2024-09-14T03:43:18.634920+0000","last_active":"2024-09-14T03:43:18.634920+0000","last_peered":"2024-09-14T03:43:18.634920+0000","last_clean":"2024-09-14T03:43:18.634920+0000","last_became_active":"2024-09-14T03:43:18.634479+0000","last_became_peered":"2024-09-14T03:43:18.634479+0000","last_unstale":"2024-09-14T03:43:18.634920+0000","last_undegraded":"2024-09-14T03:43:18.634920+0000","last_fullsized":"2024-09-14T03:43:18.634920+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:05.281377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548765+0000","last_change":"2024-09-14T03:42:34.642625+0000","last_active":"2024-09-14T03:43:17.548765+0000","last_peered":"2024-09-14T03:43:17.548765+0000","last_clean":"2024-09-14T03:43:17.548765+0000","last_became_active":"2024-09-14T03:42:34.642335+0000","last_became_peered":"2024-09-14T03:42:34.642335+0000","last_unstale":"2024-09-14T03:43:17.548765+0000","last_undegraded":"2024-09-14T03:43:17.548765+0000","last_fullsized":"2024-09-14T03:43:17.548765+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:13.909793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633413+0000","last_change":"2024-09-14T03:41:23.736609+0000","last_active":"2024-09-14T03:43:18.633413+0000","last_peered":"2024-09-14T03:43:18.633413+0000","last_clean":"2024-09-14T03:43:18.633413+0000","last_became_active":"2024-09-14T03:41:18.713287+0000","last_became_peered":"2024-09-14T03:41:18.713287+0000","last_unstale":"2024-09-14T03:43:18.633413+0000","last_undegraded":"2024-09-14T03:43:18.633413+0000","last_fullsized":"2024-09-14T03:43:18.633413+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:09.426148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545933+0000","last_change":"2024-09-14T03:42:35.827846+0000","last_active":"2024-09-14T03:43:17.545933+0000","last_peered":"2024-09-14T03:43:17.545933+0000","last_clean":"2024-09-14T03:43:17.545933+0000","last_became_active":"2024-09-14T03:42:35.827453+0000","last_became_peered":"2024-09-14T03:42:35.827453+0000","last_unstale":"2024-09-14T03:43:17.545933+0000","last_undegraded":"2024-09-14T03:43:17.545933+0000","last_fullsized":"2024-09-14T03:43:17.545933+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:00:40.709178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546772+0000","last_change":"2024-09-14T03:42:34.635824+0000","last_active":"2024-09-14T03:43:17.546772+0000","last_peered":"2024-09-14T03:43:17.546772+0000","last_clean":"2024-09-14T03:43:17.546772+0000","last_became_active":"2024-09-14T03:42:34.635531+0000","last_became_peered":"2024-09-14T03:42:34.635531+0000","last_unstale":"2024-09-14T03:43:17.546772+0000","last_undegraded":"2024-09-14T03:43:17.546772+0000","last_fullsized":"2024-09-14T03:43:17.546772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:08:15.718743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633970+0000","last_change":"2024-09-14T03:41:23.741514+0000","last_active":"2024-09-14T03:43:18.633970+0000","last_peered":"2024-09-14T03:43:18.633970+0000","last_clean":"2024-09-14T03:43:18.633970+0000","last_became_active":"2024-09-14T03:41:19.397559+0000","last_became_peered":"2024-09-14T03:41:19.397559+0000","last_unstale":"2024-09-14T03:43:18.633970+0000","last_undegraded":"2024-09-14T03:43:18.633970+0000","last_fullsized":"2024-09-14T03:43:18.633970+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:33:59.613404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148503+0000","last_change":"2024-09-14T03:42:35.827841+0000","last_active":"2024-09-14T03:43:18.148503+0000","last_peered":"2024-09-14T03:43:18.148503+0000","last_clean":"2024-09-14T03:43:18.148503+0000","last_became_active":"2024-09-14T03:42:35.827556+0000","last_became_peered":"2024-09-14T03:42:35.827556+0000","last_unstale":"2024-09-14T03:43:18.148503+0000","last_undegraded":"2024-09-14T03:43:18.148503+0000","last_fullsized":"2024-09-14T03:43:18.148503+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:27.869314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549947+0000","last_change":"2024-09-14T03:42:34.643876+0000","last_active":"2024-09-14T03:43:17.549947+0000","last_peered":"2024-09-14T03:43:17.549947+0000","last_clean":"2024-09-14T03:43:17.549947+0000","last_became_active":"2024-09-14T03:42:34.643603+0000","last_became_peered":"2024-09-14T03:42:34.643603+0000","last_unstale":"2024-09-14T03:43:17.549947+0000","last_undegraded":"2024-09-14T03:43:17.549947+0000","last_fullsized":"2024-09-14T03:43:17.549947+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:57.435387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633784+0000","last_change":"2024-09-14T03:41:23.736591+0000","last_active":"2024-09-14T03:43:18.633784+0000","last_peered":"2024-09-14T03:43:18.633784+0000","last_clean":"2024-09-14T03:43:18.633784+0000","last_became_active":"2024-09-14T03:41:19.397811+0000","last_became_peered":"2024-09-14T03:41:19.397811+0000","last_unstale":"2024-09-14T03:43:18.633784+0000","last_undegraded":"2024-09-14T03:43:18.633784+0000","last_fullsized":"2024-09-14T03:43:18.633784+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:41.233137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040775400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548328+0000","last_change":"2024-09-14T03:42:35.667323+0000","last_active":"2024-09-14T03:43:17.548328+0000","last_peered":"2024-09-14T03:43:17.548328+0000","last_clean":"2024-09-14T03:43:17.548328+0000","last_became_active":"2024-09-14T03:42:35.667019+0000","last_became_peered":"2024-09-14T03:42:35.667019+0000","last_unstale":"2024-09-14T03:43:17.548328+0000","last_undegraded":"2024-09-14T03:43:17.548328+0000","last_fullsized":"2024-09-14T03:43:17.548328+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:02.864995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148637+0000","last_change":"2024-09-14T03:42:34.648053+0000","last_active":"2024-09-14T03:43:18.148637+0000","last_peered":"2024-09-14T03:43:18.148637+0000","last_clean":"2024-09-14T03:43:18.148637+0000","last_became_active":"2024-09-14T03:42:34.647766+0000","last_became_peered":"2024-09-14T03:42:34.647766+0000","last_unstale":"2024-09-14T03:43:18.148637+0000","last_undegraded":"2024-09-14T03:43:18.148637+0000","last_fullsized":"2024-09-14T03:43:18.148637+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:00:43.804911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.547073+0000","last_change":"2024-09-14T03:41:23.740297+0000","last_active":"2024-09-14T03:43:18.547073+0000","last_peered":"2024-09-14T03:43:18.547073+0000","last_clean":"2024-09-14T03:43:18.547073+0000","last_became_active":"2024-09-14T03:41:18.713102+0000","last_became_peered":"2024-09-14T03:41:18.713102+0000","last_unstale":"2024-09-14T03:43:18.547073+0000","last_undegraded":"2024-09-14T03:43:18.547073+0000","last_fullsized":"2024-09-14T03:43:18.547073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:37:04.215608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547411+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T03:43:17.547411+0000","last_peered":"2024-09-14T03:43:17.547411+0000","last_clean":"2024-09-14T03:43:17.547411+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T03:43:17.547411+0000","last_undegraded":"2024-09-14T03:43:17.547411+0000","last_fullsized":"2024-09-14T03:43:17.547411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634965+0000","last_change":"2024-09-14T03:43:18.634965+0000","last_active":"2024-09-14T03:43:18.634965+0000","last_peered":"2024-09-14T03:43:18.634965+0000","last_clean":"2024-09-14T03:43:18.634965+0000","last_became_active":"2024-09-14T03:43:18.634451+0000","last_became_peered":"2024-09-14T03:43:18.634451+0000","last_unstale":"2024-09-14T03:43:18.634965+0000","last_undegraded":"2024-09-14T03:43:18.634965+0000","last_fullsized":"2024-09-14T03:43:18.634965+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:37.675709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547488+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T03:43:17.547488+0000","last_peered":"2024-09-14T03:43:17.547488+0000","last_clean":"2024-09-14T03:43:17.547488+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T03:43:17.547488+0000","last_undegraded":"2024-09-14T03:43:17.547488+0000","last_fullsized":"2024-09-14T03:43:17.547488+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545644+0000","last_change":"2024-09-14T03:42:35.664355+0000","last_active":"2024-09-14T03:43:17.545644+0000","last_peered":"2024-09-14T03:43:17.545644+0000","last_clean":"2024-09-14T03:43:17.545644+0000","last_became_active":"2024-09-14T03:42:35.664067+0000","last_became_peered":"2024-09-14T03:42:35.664067+0000","last_unstale":"2024-09-14T03:43:17.545644+0000","last_undegraded":"2024-09-14T03:43:17.545644+0000","last_fullsized":"2024-09-14T03:43:17.545644+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:19.133554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551235+0000","last_change":"2024-09-14T03:42:34.644211+0000","last_active":"2024-09-14T03:43:17.551235+0000","last_peered":"2024-09-14T03:43:17.551235+0000","last_clean":"2024-09-14T03:43:17.551235+0000","last_became_active":"2024-09-14T03:42:34.643937+0000","last_became_peered":"2024-09-14T03:42:34.643937+0000","last_unstale":"2024-09-14T03:43:17.551235+0000","last_undegraded":"2024-09-14T03:43:17.551235+0000","last_fullsized":"2024-09-14T03:43:17.551235+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:34.755024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548840+0000","last_change":"2024-09-14T03:42:34.640845+0000","last_active":"2024-09-14T03:43:17.548840+0000","last_peered":"2024-09-14T03:43:17.548840+0000","last_clean":"2024-09-14T03:43:17.548840+0000","last_became_active":"2024-09-14T03:42:34.640500+0000","last_became_peered":"2024-09-14T03:42:34.640500+0000","last_unstale":"2024-09-14T03:43:17.548840+0000","last_undegraded":"2024-09-14T03:43:17.548840+0000","last_fullsized":"2024-09-14T03:43:17.548840+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:54.999114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548446+0000","last_change":"2024-09-14T03:42:35.662106+0000","last_active":"2024-09-14T03:43:17.548446+0000","last_peered":"2024-09-14T03:43:17.548446+0000","last_clean":"2024-09-14T03:43:17.548446+0000","last_became_active":"2024-09-14T03:42:35.661783+0000","last_became_peered":"2024-09-14T03:42:35.661783+0000","last_unstale":"2024-09-14T03:43:17.548446+0000","last_undegraded":"2024-09-14T03:43:17.548446+0000","last_fullsized":"2024-09-14T03:43:17.548446+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:42:03.398601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148835+0000","last_change":"2024-09-14T03:42:34.648183+0000","last_active":"2024-09-14T03:43:18.148835+0000","last_peered":"2024-09-14T03:43:18.148835+0000","last_clean":"2024-09-14T03:43:18.148835+0000","last_became_active":"2024-09-14T03:42:34.647898+0000","last_became_peered":"2024-09-14T03:42:34.647898+0000","last_unstale":"2024-09-14T03:43:18.148835+0000","last_undegraded":"2024-09-14T03:43:18.148835+0000","last_fullsized":"2024-09-14T03:43:18.148835+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:44.598862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549294+0000","last_change":"2024-09-14T03:42:35.664361+0000","last_active":"2024-09-14T03:43:17.549294+0000","last_peered":"2024-09-14T03:43:17.549294+0000","last_clean":"2024-09-14T03:43:17.549294+0000","last_became_active":"2024-09-14T03:42:35.664074+0000","last_became_peered":"2024-09-14T03:42:35.664074+0000","last_unstale":"2024-09-14T03:43:17.549294+0000","last_undegraded":"2024-09-14T03:43:17.549294+0000","last_fullsized":"2024-09-14T03:43:17.549294+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:16.964809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550605+0000","last_change":"2024-09-14T03:42:35.827946+0000","last_active":"2024-09-14T03:43:17.550605+0000","last_peered":"2024-09-14T03:43:17.550605+0000","last_clean":"2024-09-14T03:43:17.550605+0000","last_became_active":"2024-09-14T03:42:35.827619+0000","last_became_peered":"2024-09-14T03:42:35.827619+0000","last_unstale":"2024-09-14T03:43:17.550605+0000","last_undegraded":"2024-09-14T03:43:17.550605+0000","last_fullsized":"2024-09-14T03:43:17.550605+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:35.250158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548436+0000","last_change":"2024-09-14T03:42:34.641678+0000","last_active":"2024-09-14T03:43:17.548436+0000","last_peered":"2024-09-14T03:43:17.548436+0000","last_clean":"2024-09-14T03:43:17.548436+0000","last_became_active":"2024-09-14T03:42:34.641420+0000","last_became_peered":"2024-09-14T03:42:34.641420+0000","last_unstale":"2024-09-14T03:43:17.548436+0000","last_undegraded":"2024-09-14T03:43:17.548436+0000","last_fullsized":"2024-09-14T03:43:17.548436+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:00.255578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148440+0000","last_change":"2024-09-14T03:42:35.665205+0000","last_active":"2024-09-14T03:43:18.148440+0000","last_peered":"2024-09-14T03:43:18.148440+0000","last_clean":"2024-09-14T03:43:18.148440+0000","last_became_active":"2024-09-14T03:42:35.664934+0000","last_became_peered":"2024-09-14T03:42:35.664934+0000","last_unstale":"2024-09-14T03:43:18.148440+0000","last_undegraded":"2024-09-14T03:43:18.148440+0000","last_fullsized":"2024-09-14T03:43:18.148440+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:13:21.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545130+0000","last_change":"2024-09-14T03:42:34.650983+0000","last_active":"2024-09-14T03:43:17.545130+0000","last_peered":"2024-09-14T03:43:17.545130+0000","last_clean":"2024-09-14T03:43:17.545130+0000","last_became_active":"2024-09-14T03:42:34.650690+0000","last_became_peered":"2024-09-14T03:42:34.650690+0000","last_unstale":"2024-09-14T03:43:17.545130+0000","last_undegraded":"2024-09-14T03:43:17.545130+0000","last_fullsized":"2024-09-14T03:43:17.545130+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:59.946101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549518+0000","last_change":"2024-09-14T03:42:35.666729+0000","last_active":"2024-09-14T03:43:17.549518+0000","last_peered":"2024-09-14T03:43:17.549518+0000","last_clean":"2024-09-14T03:43:17.549518+0000","last_became_active":"2024-09-14T03:42:35.666452+0000","last_became_peered":"2024-09-14T03:42:35.666452+0000","last_unstale":"2024-09-14T03:43:17.549518+0000","last_undegraded":"2024-09-14T03:43:17.549518+0000","last_fullsized":"2024-09-14T03:43:17.549518+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:25.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548183+0000","last_change":"2024-09-14T03:42:34.651650+0000","last_active":"2024-09-14T03:43:17.548183+0000","last_peered":"2024-09-14T03:43:17.548183+0000","last_clean":"2024-09-14T03:43:17.548183+0000","last_became_active":"2024-09-14T03:42:34.651389+0000","last_became_peered":"2024-09-14T03:42:34.651389+0000","last_unstale":"2024-09-14T03:43:17.548183+0000","last_undegraded":"2024-09-14T03:43:17.548183+0000","last_fullsized":"2024-09-14T03:43:17.548183+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:28.695287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546030+0000","last_change":"2024-09-14T03:42:34.663365+0000","last_active":"2024-09-14T03:43:17.546030+0000","last_peered":"2024-09-14T03:43:17.546030+0000","last_clean":"2024-09-14T03:43:17.546030+0000","last_became_active":"2024-09-14T03:42:34.663042+0000","last_became_peered":"2024-09-14T03:42:34.663042+0000","last_unstale":"2024-09-14T03:43:17.546030+0000","last_undegraded":"2024-09-14T03:43:17.546030+0000","last_fullsized":"2024-09-14T03:43:17.546030+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:29:06.456493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148585+0000","last_change":"2024-09-14T03:42:34.650367+0000","last_active":"2024-09-14T03:43:18.148585+0000","last_peered":"2024-09-14T03:43:18.148585+0000","last_clean":"2024-09-14T03:43:18.148585+0000","last_became_active":"2024-09-14T03:42:34.650086+0000","last_became_peered":"2024-09-14T03:42:34.650086+0000","last_unstale":"2024-09-14T03:43:18.148585+0000","last_undegraded":"2024-09-14T03:43:18.148585+0000","last_fullsized":"2024-09-14T03:43:18.148585+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:26.436325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547497+0000","last_change":"2024-09-14T03:42:35.664722+0000","last_active":"2024-09-14T03:43:17.547497+0000","last_peered":"2024-09-14T03:43:17.547497+0000","last_clean":"2024-09-14T03:43:17.547497+0000","last_became_active":"2024-09-14T03:42:35.664150+0000","last_became_peered":"2024-09-14T03:42:35.664150+0000","last_unstale":"2024-09-14T03:43:17.547497+0000","last_undegraded":"2024-09-14T03:43:17.547497+0000","last_fullsized":"2024-09-14T03:43:17.547497+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:31.543679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548706+0000","last_change":"2024-09-14T03:42:34.646474+0000","last_active":"2024-09-14T03:43:17.548706+0000","last_peered":"2024-09-14T03:43:17.548706+0000","last_clean":"2024-09-14T03:43:17.548706+0000","last_became_active":"2024-09-14T03:42:34.646139+0000","last_became_peered":"2024-09-14T03:42:34.646139+0000","last_unstale":"2024-09-14T03:43:17.548706+0000","last_undegraded":"2024-09-14T03:43:17.548706+0000","last_fullsized":"2024-09-14T03:43:17.548706+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:07.526545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547542+0000","last_change":"2024-09-14T03:42:35.661687+0000","last_active":"2024-09-14T03:43:17.547542+0000","last_peered":"2024-09-14T03:43:17.547542+0000","last_clean":"2024-09-14T03:43:17.547542+0000","last_became_active":"2024-09-14T03:42:35.661155+0000","last_became_peered":"2024-09-14T03:42:35.661155+0000","last_unstale":"2024-09-14T03:43:17.547542+0000","last_undegraded":"2024-09-14T03:43:17.547542+0000","last_fullsized":"2024-09-14T03:43:17.547542+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:16.038383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548715+0000","last_change":"2024-09-14T03:42:34.645745+0000","last_active":"2024-09-14T03:43:17.548715+0000","last_peered":"2024-09-14T03:43:17.548715+0000","last_clean":"2024-09-14T03:43:17.548715+0000","last_became_active":"2024-09-14T03:42:34.645455+0000","last_became_peered":"2024-09-14T03:42:34.645455+0000","last_unstale":"2024-09-14T03:43:17.548715+0000","last_undegraded":"2024-09-14T03:43:17.548715+0000","last_fullsized":"2024-09-14T03:43:17.548715+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:19.729438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548642+0000","last_change":"2024-09-14T03:42:35.666516+0000","last_active":"2024-09-14T03:43:17.548642+0000","last_peered":"2024-09-14T03:43:17.548642+0000","last_clean":"2024-09-14T03:43:17.548642+0000","last_became_active":"2024-09-14T03:42:35.666172+0000","last_became_peered":"2024-09-14T03:42:35.666172+0000","last_unstale":"2024-09-14T03:43:17.548642+0000","last_undegraded":"2024-09-14T03:43:17.548642+0000","last_fullsized":"2024-09-14T03:43:17.548642+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:57:25.139258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549760+0000","last_change":"2024-09-14T03:42:34.640794+0000","last_active":"2024-09-14T03:43:17.549760+0000","last_peered":"2024-09-14T03:43:17.549760+0000","last_clean":"2024-09-14T03:43:17.549760+0000","last_became_active":"2024-09-14T03:42:34.640417+0000","last_became_peered":"2024-09-14T03:42:34.640417+0000","last_unstale":"2024-09-14T03:43:17.549760+0000","last_undegraded":"2024-09-14T03:43:17.549760+0000","last_fullsized":"2024-09-14T03:43:17.549760+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:36.857651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547109+0000","last_change":"2024-09-14T03:42:35.664713+0000","last_active":"2024-09-14T03:43:17.547109+0000","last_peered":"2024-09-14T03:43:17.547109+0000","last_clean":"2024-09-14T03:43:17.547109+0000","last_became_active":"2024-09-14T03:42:35.664138+0000","last_became_peered":"2024-09-14T03:42:35.664138+0000","last_unstale":"2024-09-14T03:43:17.547109+0000","last_undegraded":"2024-09-14T03:43:17.547109+0000","last_fullsized":"2024-09-14T03:43:17.547109+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:16:31.298143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148751+0000","last_change":"2024-09-14T03:42:34.651887+0000","last_active":"2024-09-14T03:43:18.148751+0000","last_peered":"2024-09-14T03:43:18.148751+0000","last_clean":"2024-09-14T03:43:18.148751+0000","last_became_active":"2024-09-14T03:42:34.650986+0000","last_became_peered":"2024-09-14T03:42:34.650986+0000","last_unstale":"2024-09-14T03:43:18.148751+0000","last_undegraded":"2024-09-14T03:43:18.148751+0000","last_fullsized":"2024-09-14T03:43:18.148751+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:29.067192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28380,"kb_used_data":1356,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716772,"statfs":{"total":95995035648,"available":95965974528,"internally_reserved":0,"allocated":1388544,"data_stored":928583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.49199999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.223,"15min":0.223},"max":{"1min":0.66600000000000004,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.72799999999999998,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.56699999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.63500000000000001,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.33900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.68100000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72699999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.435,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.71999999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.70399999999999996}]},{"osd":4,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.47699999999999998,"15min":0.48499999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.5,"15min":0.503},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.52600000000000002,"15min":0.53200000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67400000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73699999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55200000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32456,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712696,"statfs":{"total":95995035648,"available":95961800704,"internally_reserved":0,"allocated":1372160,"data_stored":927902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54200000000000004,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.71999999999999997,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56599999999999995,"15min":0.57399999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77200000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72399999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.54500000000000004,"15min":0.55500000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78600000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70699999999999996,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76700000000000002,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.44600000000000001,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.437},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.46400000000000002,"15min":0.442},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72999999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.625}]},{"osd":5,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48399999999999999,"15min":0.46200000000000002},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73199999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.501,"15min":0.48499999999999999},"min":{"1min":0.32800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.53400000000000003,"15min":0.52200000000000002},"min":{"1min":0.39200000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53200000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":774144,"data_stored":336825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.35899999999999999,"5min":0.31,"15min":0.31},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.755,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.44400000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81499999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85299999999999998,"5min":0.879,"15min":0.879},"last":0.68400000000000005}]},{"osd":4,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.39600000000000002},"min":{"1min":0.25,"5min":0.246,"15min":0.246},"max":{"1min":0.71999999999999997,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.434},{"interface":"front","average":{"1min":0.442,"5min":0.41499999999999998,"15min":0.40200000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:42:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.47399999999999998,"15min":0.46500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.47299999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.876,"15min":0.876},"last":0.64700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27792,"kb_used_data":768,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717360,"statfs":{"total":95995035648,"available":95966576640,"internally_reserved":0,"allocated":786432,"data_stored":337872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.73999999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.52300000000000002,"15min":0.52500000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.67600000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.501}]},{"osd":1,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53800000000000003,"15min":0.54500000000000004},"min":{"1min":0.38200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77200000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.69099999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73699999999999999,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80300000000000005,"5min":0.87,"15min":0.87},"last":0.434},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.38500000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.67700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.41799999999999998,"15min":0.40200000000000002},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45300000000000001,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72799999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:43:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41699999999999998,"15min":0.39100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.71799999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.50600000000000001,"15min":0.47299999999999998},"min":{"1min":0.39300000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79100000000000004,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48099999999999998,"15min":0.47499999999999998},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.78200000000000003,"5min":1.008,"15min":1.008},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.48299999999999998,"15min":0.48099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.66800000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":770048,"data_stored":336525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.38700000000000001,"15min":0.39700000000000002},"min":{"1min":0.215,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51200000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.41099999999999998,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.43099999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.48599999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.439,"15min":0.44800000000000001},"min":{"1min":0.26200000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.64200000000000002,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.437}]},{"osd":2,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.46000000000000002,"15min":0.47799999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54500000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41399999999999998,"15min":0.42199999999999999},"min":{"1min":0.29199999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.373,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.72799999999999998,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.39900000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72099999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.58599999999999997,"15min":0.57999999999999996},"min":{"1min":0.42199999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81499999999999995,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63600000000000001,"15min":0.64400000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74299999999999999,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.76500000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77400000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.64500000000000002,"15min":0.64900000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.67900000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31876,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713276,"statfs":{"total":95995035648,"available":95962394624,"internally_reserved":0,"allocated":774144,"data_stored":337051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38,"15min":0.379},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.47299999999999998,"5min":1.244,"15min":1.244},"last":0.318},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.373,"15min":0.37},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.58699999999999997,"5min":1.204,"15min":1.204},"last":0.371},{"interface":"front","average":{"1min":0.377,"5min":0.39000000000000001,"15min":0.39900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.53800000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41999999999999998,"15min":0.42299999999999999},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.42199999999999999,"15min":0.44500000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.50900000000000001,"5min":0.629,"15min":0.629},"last":0.441}]},{"osd":4,"last update":"Sat Sep 14 03:42:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.45300000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78900000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58299999999999996,"15min":0.57099999999999995},"min":{"1min":0.46600000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.76300000000000001,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.73999999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60199999999999998,"15min":0.61799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74299999999999999,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.629,"15min":0.63},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.80900000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.621,"15min":0.627},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73199999999999998,"5min":1.333,"15min":1.333},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.77300000000000002,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.68400000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":336454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.32700000000000001},"min":{"1min":0.20599999999999999,"5min":0.185,"15min":0.185},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.32900000000000001},"min":{"1min":0.26800000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36799999999999999,"15min":0.36099999999999999},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:43:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68700000000000006,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.66900000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 14 03:42:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59399999999999997,"15min":0.61199999999999999},"min":{"1min":0.436,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55700000000000005,"15min":0.56899999999999995},"min":{"1min":0.375,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Sep 14 03:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.622,"5min":0.63800000000000001,"15min":0.65400000000000003},"min":{"1min":0.42499999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.871,"15min":0.871},"last":0.81200000000000006}]},{"osd":6,"last update":"Sat Sep 14 03:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77600000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.72299999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.61099999999999999,"15min":0.61499999999999999},"min":{"1min":0.43099999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78900000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.88600000000000001,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.77900000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32456,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712696,"statfs":{"total":95995035648,"available":95961800704,"internally_reserved":0,"allocated":1368064,"data_stored":927407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.33400000000000002},"min":{"1min":0.28000000000000003,"5min":0.224,"15min":0.224},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.373,"15min":0.33200000000000002},"min":{"1min":0.27300000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.48899999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:43:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39100000000000001,"15min":0.378},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.55500000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.39300000000000002,"15min":0.371},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.57099999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.307}]},{"osd":3,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.44600000000000001,"15min":0.46800000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.52600000000000002,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.46300000000000002,"15min":0.48099999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68300000000000005,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.371}]},{"osd":4,"last update":"Sat Sep 14 03:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.752,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57599999999999996,"15min":0.56799999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.872,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84199999999999997,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.502}]},{"osd":6,"last update":"Sat Sep 14 03:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.65300000000000002,"15min":0.66100000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.88400000000000001,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.67000000000000004,"15min":0.68400000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58499999999999996,"15min":0.59099999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84499999999999997,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.64500000000000002,"15min":0.65300000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:43:33.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:43:33.916 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:34 smithi001 ceph-mon[29360]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:34 smithi001 ceph-mon[29360]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:34 smithi001 ceph-mon[40972]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:34 smithi001 ceph-mon[40972]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:34 smithi012 ceph-mon[33640]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:34 smithi012 ceph-mon[33640]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:35.737 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:43:35.738 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T03:43:36.336 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-09-14T03:43:34.693902+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":240380,"kb_used_data":7816,"kb_used_omap":12,"kb_used_meta":232435,"kb_avail":749720836,"statfs":{"total":767960285184,"available":767714136064,"internally_reserved":0,"allocated":8003584,"data_stored":4468619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":238014008},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002573"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634992+0000","last_change":"2024-09-14T03:43:18.634992+0000","last_active":"2024-09-14T03:43:18.634992+0000","last_peered":"2024-09-14T03:43:18.634992+0000","last_clean":"2024-09-14T03:43:18.634992+0000","last_became_active":"2024-09-14T03:43:18.634473+0000","last_became_peered":"2024-09-14T03:43:18.634473+0000","last_unstale":"2024-09-14T03:43:18.634992+0000","last_undegraded":"2024-09-14T03:43:18.634992+0000","last_fullsized":"2024-09-14T03:43:18.634992+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:32.160322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551646+0000","last_change":"2024-09-14T03:42:34.647856+0000","last_active":"2024-09-14T03:43:17.551646+0000","last_peered":"2024-09-14T03:43:17.551646+0000","last_clean":"2024-09-14T03:43:17.551646+0000","last_became_active":"2024-09-14T03:42:34.647566+0000","last_became_peered":"2024-09-14T03:42:34.647566+0000","last_unstale":"2024-09-14T03:43:17.551646+0000","last_undegraded":"2024-09-14T03:43:17.551646+0000","last_fullsized":"2024-09-14T03:43:17.551646+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:35.220436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548652+0000","last_change":"2024-09-14T03:42:34.882870+0000","last_active":"2024-09-14T03:43:17.548652+0000","last_peered":"2024-09-14T03:43:17.548652+0000","last_clean":"2024-09-14T03:43:17.548652+0000","last_became_active":"2024-09-14T03:42:34.882555+0000","last_became_peered":"2024-09-14T03:42:34.882555+0000","last_unstale":"2024-09-14T03:43:17.548652+0000","last_undegraded":"2024-09-14T03:43:17.548652+0000","last_fullsized":"2024-09-14T03:43:17.548652+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:34.325899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148346+0000","last_change":"2024-09-14T03:42:34.649795+0000","last_active":"2024-09-14T03:43:18.148346+0000","last_peered":"2024-09-14T03:43:18.148346+0000","last_clean":"2024-09-14T03:43:18.148346+0000","last_became_active":"2024-09-14T03:42:34.649531+0000","last_became_peered":"2024-09-14T03:42:34.649531+0000","last_unstale":"2024-09-14T03:43:18.148346+0000","last_undegraded":"2024-09-14T03:43:18.148346+0000","last_fullsized":"2024-09-14T03:43:18.148346+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:01.527597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148589+0000","last_change":"2024-09-14T03:42:35.664546+0000","last_active":"2024-09-14T03:43:18.148589+0000","last_peered":"2024-09-14T03:43:18.148589+0000","last_clean":"2024-09-14T03:43:18.148589+0000","last_became_active":"2024-09-14T03:42:35.664287+0000","last_became_peered":"2024-09-14T03:42:35.664287+0000","last_unstale":"2024-09-14T03:43:18.148589+0000","last_undegraded":"2024-09-14T03:43:18.148589+0000","last_fullsized":"2024-09-14T03:43:18.148589+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:11.241309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551248+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T03:43:17.551248+0000","last_peered":"2024-09-14T03:43:17.551248+0000","last_clean":"2024-09-14T03:43:17.551248+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T03:43:17.551248+0000","last_undegraded":"2024-09-14T03:43:17.551248+0000","last_fullsized":"2024-09-14T03:43:17.551248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549689+0000","last_change":"2024-09-14T03:42:35.665434+0000","last_active":"2024-09-14T03:43:17.549689+0000","last_peered":"2024-09-14T03:43:17.549689+0000","last_clean":"2024-09-14T03:43:17.549689+0000","last_became_active":"2024-09-14T03:42:35.665037+0000","last_became_peered":"2024-09-14T03:42:35.665037+0000","last_unstale":"2024-09-14T03:43:17.549689+0000","last_undegraded":"2024-09-14T03:43:17.549689+0000","last_fullsized":"2024-09-14T03:43:17.549689+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:06.613986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550233+0000","last_change":"2024-09-14T03:42:34.644354+0000","last_active":"2024-09-14T03:43:17.550233+0000","last_peered":"2024-09-14T03:43:17.550233+0000","last_clean":"2024-09-14T03:43:17.550233+0000","last_became_active":"2024-09-14T03:42:34.644093+0000","last_became_peered":"2024-09-14T03:42:34.644093+0000","last_unstale":"2024-09-14T03:43:17.550233+0000","last_undegraded":"2024-09-14T03:43:17.550233+0000","last_fullsized":"2024-09-14T03:43:17.550233+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:07.595587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551942+0000","last_change":"2024-09-14T03:42:34.648183+0000","last_active":"2024-09-14T03:43:17.551942+0000","last_peered":"2024-09-14T03:43:17.551942+0000","last_clean":"2024-09-14T03:43:17.551942+0000","last_became_active":"2024-09-14T03:42:34.647786+0000","last_became_peered":"2024-09-14T03:42:34.647786+0000","last_unstale":"2024-09-14T03:43:17.551942+0000","last_undegraded":"2024-09-14T03:43:17.551942+0000","last_fullsized":"2024-09-14T03:43:17.551942+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:30:45.555570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548255+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T03:43:17.548255+0000","last_peered":"2024-09-14T03:43:17.548255+0000","last_clean":"2024-09-14T03:43:17.548255+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T03:43:17.548255+0000","last_undegraded":"2024-09-14T03:43:17.548255+0000","last_fullsized":"2024-09-14T03:43:17.548255+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551883+0000","last_change":"2024-09-14T03:42:34.643514+0000","last_active":"2024-09-14T03:43:17.551883+0000","last_peered":"2024-09-14T03:43:17.551883+0000","last_clean":"2024-09-14T03:43:17.551883+0000","last_became_active":"2024-09-14T03:42:34.643235+0000","last_became_peered":"2024-09-14T03:42:34.643235+0000","last_unstale":"2024-09-14T03:43:17.551883+0000","last_undegraded":"2024-09-14T03:43:17.551883+0000","last_fullsized":"2024-09-14T03:43:17.551883+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:26.173830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547137+0000","last_change":"2024-09-14T03:42:35.664330+0000","last_active":"2024-09-14T03:43:17.547137+0000","last_peered":"2024-09-14T03:43:17.547137+0000","last_clean":"2024-09-14T03:43:17.547137+0000","last_became_active":"2024-09-14T03:42:35.664023+0000","last_became_peered":"2024-09-14T03:42:35.664023+0000","last_unstale":"2024-09-14T03:43:17.547137+0000","last_undegraded":"2024-09-14T03:43:17.547137+0000","last_fullsized":"2024-09-14T03:43:17.547137+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:30.956305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551763+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T03:43:17.551763+0000","last_peered":"2024-09-14T03:43:17.551763+0000","last_clean":"2024-09-14T03:43:17.551763+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T03:43:17.551763+0000","last_undegraded":"2024-09-14T03:43:17.551763+0000","last_fullsized":"2024-09-14T03:43:17.551763+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550269+0000","last_change":"2024-09-14T03:42:35.664374+0000","last_active":"2024-09-14T03:43:17.550269+0000","last_peered":"2024-09-14T03:43:17.550269+0000","last_clean":"2024-09-14T03:43:17.550269+0000","last_became_active":"2024-09-14T03:42:35.664099+0000","last_became_peered":"2024-09-14T03:42:35.664099+0000","last_unstale":"2024-09-14T03:43:17.550269+0000","last_undegraded":"2024-09-14T03:43:17.550269+0000","last_fullsized":"2024-09-14T03:43:17.550269+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:31.081575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548825+0000","last_change":"2024-09-14T03:42:34.641776+0000","last_active":"2024-09-14T03:43:17.548825+0000","last_peered":"2024-09-14T03:43:17.548825+0000","last_clean":"2024-09-14T03:43:17.548825+0000","last_became_active":"2024-09-14T03:42:34.641470+0000","last_became_peered":"2024-09-14T03:42:34.641470+0000","last_unstale":"2024-09-14T03:43:17.548825+0000","last_undegraded":"2024-09-14T03:43:17.548825+0000","last_fullsized":"2024-09-14T03:43:17.548825+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.940502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148406+0000","last_change":"2024-09-14T03:42:35.666256+0000","last_active":"2024-09-14T03:43:18.148406+0000","last_peered":"2024-09-14T03:43:18.148406+0000","last_clean":"2024-09-14T03:43:18.148406+0000","last_became_active":"2024-09-14T03:42:35.665990+0000","last_became_peered":"2024-09-14T03:42:35.665990+0000","last_unstale":"2024-09-14T03:43:18.148406+0000","last_undegraded":"2024-09-14T03:43:18.148406+0000","last_fullsized":"2024-09-14T03:43:18.148406+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:37.777147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549149+0000","last_change":"2024-09-14T03:42:34.643600+0000","last_active":"2024-09-14T03:43:17.549149+0000","last_peered":"2024-09-14T03:43:17.549149+0000","last_clean":"2024-09-14T03:43:17.549149+0000","last_became_active":"2024-09-14T03:42:34.643303+0000","last_became_peered":"2024-09-14T03:42:34.643303+0000","last_unstale":"2024-09-14T03:43:17.549149+0000","last_undegraded":"2024-09-14T03:43:17.549149+0000","last_fullsized":"2024-09-14T03:43:17.549149+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:20.221506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547317+0000","last_change":"2024-09-14T03:42:35.661473+0000","last_active":"2024-09-14T03:43:17.547317+0000","last_peered":"2024-09-14T03:43:17.547317+0000","last_clean":"2024-09-14T03:43:17.547317+0000","last_became_active":"2024-09-14T03:42:35.661188+0000","last_became_peered":"2024-09-14T03:42:35.661188+0000","last_unstale":"2024-09-14T03:43:17.547317+0000","last_undegraded":"2024-09-14T03:43:17.547317+0000","last_fullsized":"2024-09-14T03:43:17.547317+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:51:30.698952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550482+0000","last_change":"2024-09-14T03:42:34.646942+0000","last_active":"2024-09-14T03:43:17.550482+0000","last_peered":"2024-09-14T03:43:17.550482+0000","last_clean":"2024-09-14T03:43:17.550482+0000","last_became_active":"2024-09-14T03:42:34.646614+0000","last_became_peered":"2024-09-14T03:42:34.646614+0000","last_unstale":"2024-09-14T03:43:17.550482+0000","last_undegraded":"2024-09-14T03:43:17.550482+0000","last_fullsized":"2024-09-14T03:43:17.550482+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:21:15.407821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547841+0000","last_change":"2024-09-14T03:42:35.664990+0000","last_active":"2024-09-14T03:43:17.547841+0000","last_peered":"2024-09-14T03:43:17.547841+0000","last_clean":"2024-09-14T03:43:17.547841+0000","last_became_active":"2024-09-14T03:42:35.664644+0000","last_became_peered":"2024-09-14T03:42:35.664644+0000","last_unstale":"2024-09-14T03:43:17.547841+0000","last_undegraded":"2024-09-14T03:43:17.547841+0000","last_fullsized":"2024-09-14T03:43:17.547841+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:58.814241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547857+0000","last_change":"2024-09-14T03:42:34.650432+0000","last_active":"2024-09-14T03:43:17.547857+0000","last_peered":"2024-09-14T03:43:17.547857+0000","last_clean":"2024-09-14T03:43:17.547857+0000","last_became_active":"2024-09-14T03:42:34.650103+0000","last_became_peered":"2024-09-14T03:42:34.650103+0000","last_unstale":"2024-09-14T03:43:17.547857+0000","last_undegraded":"2024-09-14T03:43:17.547857+0000","last_fullsized":"2024-09-14T03:43:17.547857+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:12.350046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547756+0000","last_change":"2024-09-14T03:42:35.664372+0000","last_active":"2024-09-14T03:43:17.547756+0000","last_peered":"2024-09-14T03:43:17.547756+0000","last_clean":"2024-09-14T03:43:17.547756+0000","last_became_active":"2024-09-14T03:42:35.664063+0000","last_became_peered":"2024-09-14T03:42:35.664063+0000","last_unstale":"2024-09-14T03:43:17.547756+0000","last_undegraded":"2024-09-14T03:43:17.547756+0000","last_fullsized":"2024-09-14T03:43:17.547756+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:12:19.444102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548535+0000","last_change":"2024-09-14T03:42:34.643342+0000","last_active":"2024-09-14T03:43:17.548535+0000","last_peered":"2024-09-14T03:43:17.548535+0000","last_clean":"2024-09-14T03:43:17.548535+0000","last_became_active":"2024-09-14T03:42:34.643059+0000","last_became_peered":"2024-09-14T03:42:34.643059+0000","last_unstale":"2024-09-14T03:43:17.548535+0000","last_undegraded":"2024-09-14T03:43:17.548535+0000","last_fullsized":"2024-09-14T03:43:17.548535+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:29.317772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546136+0000","last_change":"2024-09-14T03:42:35.667459+0000","last_active":"2024-09-14T03:43:17.546136+0000","last_peered":"2024-09-14T03:43:17.546136+0000","last_clean":"2024-09-14T03:43:17.546136+0000","last_became_active":"2024-09-14T03:42:35.667192+0000","last_became_peered":"2024-09-14T03:42:35.667192+0000","last_unstale":"2024-09-14T03:43:17.546136+0000","last_undegraded":"2024-09-14T03:43:17.546136+0000","last_fullsized":"2024-09-14T03:43:17.546136+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:31.239739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":270,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551520+0000","last_change":"2024-09-14T03:40:51.032125+0000","last_active":"2024-09-14T03:43:17.551520+0000","last_peered":"2024-09-14T03:43:17.551520+0000","last_clean":"2024-09-14T03:43:17.551520+0000","last_became_active":"2024-09-14T03:40:51.031753+0000","last_became_peered":"2024-09-14T03:40:51.031753+0000","last_unstale":"2024-09-14T03:43:17.551520+0000","last_undegraded":"2024-09-14T03:43:17.551520+0000","last_fullsized":"2024-09-14T03:43:17.551520+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:30:14.411793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.550021+0000","last_change":"2024-09-14T03:41:23.740334+0000","last_active":"2024-09-14T03:43:18.550021+0000","last_peered":"2024-09-14T03:43:18.550021+0000","last_clean":"2024-09-14T03:43:18.550021+0000","last_became_active":"2024-09-14T03:41:18.712934+0000","last_became_peered":"2024-09-14T03:41:18.712934+0000","last_unstale":"2024-09-14T03:43:18.550021+0000","last_undegraded":"2024-09-14T03:43:18.550021+0000","last_fullsized":"2024-09-14T03:43:18.550021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:30:42.245712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548966+0000","last_change":"2024-09-14T03:42:34.640923+0000","last_active":"2024-09-14T03:43:17.548966+0000","last_peered":"2024-09-14T03:43:17.548966+0000","last_clean":"2024-09-14T03:43:17.548966+0000","last_became_active":"2024-09-14T03:42:34.640629+0000","last_became_peered":"2024-09-14T03:42:34.640629+0000","last_unstale":"2024-09-14T03:43:17.548966+0000","last_undegraded":"2024-09-14T03:43:17.548966+0000","last_fullsized":"2024-09-14T03:43:17.548966+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:45.483399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550716+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T03:43:17.550716+0000","last_peered":"2024-09-14T03:43:17.550716+0000","last_clean":"2024-09-14T03:43:17.550716+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T03:43:17.550716+0000","last_undegraded":"2024-09-14T03:43:17.550716+0000","last_fullsized":"2024-09-14T03:43:17.550716+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633310+0000","last_change":"2024-09-14T03:41:23.741581+0000","last_active":"2024-09-14T03:43:18.633310+0000","last_peered":"2024-09-14T03:43:18.633310+0000","last_clean":"2024-09-14T03:43:18.633310+0000","last_became_active":"2024-09-14T03:41:18.714655+0000","last_became_peered":"2024-09-14T03:41:18.714655+0000","last_unstale":"2024-09-14T03:43:18.633310+0000","last_undegraded":"2024-09-14T03:43:18.633310+0000","last_fullsized":"2024-09-14T03:43:18.633310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:56.242065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148205+0000","last_change":"2024-09-14T03:42:34.648700+0000","last_active":"2024-09-14T03:43:18.148205+0000","last_peered":"2024-09-14T03:43:18.148205+0000","last_clean":"2024-09-14T03:43:18.148205+0000","last_became_active":"2024-09-14T03:42:34.648392+0000","last_became_peered":"2024-09-14T03:42:34.648392+0000","last_unstale":"2024-09-14T03:43:18.148205+0000","last_undegraded":"2024-09-14T03:43:18.148205+0000","last_fullsized":"2024-09-14T03:43:18.148205+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:21.595422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148547+0000","last_change":"2024-09-14T03:42:35.669177+0000","last_active":"2024-09-14T03:43:18.148547+0000","last_peered":"2024-09-14T03:43:18.148547+0000","last_clean":"2024-09-14T03:43:18.148547+0000","last_became_active":"2024-09-14T03:42:35.668885+0000","last_became_peered":"2024-09-14T03:42:35.668885+0000","last_unstale":"2024-09-14T03:43:18.148547+0000","last_undegraded":"2024-09-14T03:43:18.148547+0000","last_fullsized":"2024-09-14T03:43:18.148547+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:08.424889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633512+0000","last_change":"2024-09-14T03:41:23.737002+0000","last_active":"2024-09-14T03:43:18.633512+0000","last_peered":"2024-09-14T03:43:18.633512+0000","last_clean":"2024-09-14T03:43:18.633512+0000","last_became_active":"2024-09-14T03:41:18.715814+0000","last_became_peered":"2024-09-14T03:41:18.715814+0000","last_unstale":"2024-09-14T03:43:18.633512+0000","last_undegraded":"2024-09-14T03:43:18.633512+0000","last_fullsized":"2024-09-14T03:43:18.633512+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:04.446799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.544922+0000","last_change":"2024-09-14T03:42:34.636512+0000","last_active":"2024-09-14T03:43:17.544922+0000","last_peered":"2024-09-14T03:43:17.544922+0000","last_clean":"2024-09-14T03:43:17.544922+0000","last_became_active":"2024-09-14T03:42:34.636117+0000","last_became_peered":"2024-09-14T03:42:34.636117+0000","last_unstale":"2024-09-14T03:43:17.544922+0000","last_undegraded":"2024-09-14T03:43:17.544922+0000","last_fullsized":"2024-09-14T03:43:17.544922+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:58.012169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546217+0000","last_change":"2024-09-14T03:42:35.827893+0000","last_active":"2024-09-14T03:43:17.546217+0000","last_peered":"2024-09-14T03:43:17.546217+0000","last_clean":"2024-09-14T03:43:17.546217+0000","last_became_active":"2024-09-14T03:42:35.827559+0000","last_became_peered":"2024-09-14T03:42:35.827559+0000","last_unstale":"2024-09-14T03:43:17.546217+0000","last_undegraded":"2024-09-14T03:43:17.546217+0000","last_fullsized":"2024-09-14T03:43:17.546217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:29:31.099286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":30,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.549849+0000","last_change":"2024-09-14T03:41:23.740892+0000","last_active":"2024-09-14T03:43:18.549849+0000","last_peered":"2024-09-14T03:43:18.549849+0000","last_clean":"2024-09-14T03:43:18.549849+0000","last_became_active":"2024-09-14T03:41:18.713385+0000","last_became_peered":"2024-09-14T03:41:18.713385+0000","last_unstale":"2024-09-14T03:43:18.549849+0000","last_undegraded":"2024-09-14T03:43:18.549849+0000","last_fullsized":"2024-09-14T03:43:18.549849+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:58.288379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548492+0000","last_change":"2024-09-14T03:42:34.650038+0000","last_active":"2024-09-14T03:43:17.548492+0000","last_peered":"2024-09-14T03:43:17.548492+0000","last_clean":"2024-09-14T03:43:17.548492+0000","last_became_active":"2024-09-14T03:42:34.649777+0000","last_became_peered":"2024-09-14T03:42:34.649777+0000","last_unstale":"2024-09-14T03:43:17.548492+0000","last_undegraded":"2024-09-14T03:43:17.548492+0000","last_fullsized":"2024-09-14T03:43:17.548492+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:03:45.718863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634920+0000","last_change":"2024-09-14T03:43:18.634920+0000","last_active":"2024-09-14T03:43:18.634920+0000","last_peered":"2024-09-14T03:43:18.634920+0000","last_clean":"2024-09-14T03:43:18.634920+0000","last_became_active":"2024-09-14T03:43:18.634479+0000","last_became_peered":"2024-09-14T03:43:18.634479+0000","last_unstale":"2024-09-14T03:43:18.634920+0000","last_undegraded":"2024-09-14T03:43:18.634920+0000","last_fullsized":"2024-09-14T03:43:18.634920+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:05.281377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548765+0000","last_change":"2024-09-14T03:42:34.642625+0000","last_active":"2024-09-14T03:43:17.548765+0000","last_peered":"2024-09-14T03:43:17.548765+0000","last_clean":"2024-09-14T03:43:17.548765+0000","last_became_active":"2024-09-14T03:42:34.642335+0000","last_became_peered":"2024-09-14T03:42:34.642335+0000","last_unstale":"2024-09-14T03:43:17.548765+0000","last_undegraded":"2024-09-14T03:43:17.548765+0000","last_fullsized":"2024-09-14T03:43:17.548765+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:13.909793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633413+0000","last_change":"2024-09-14T03:41:23.736609+0000","last_active":"2024-09-14T03:43:18.633413+0000","last_peered":"2024-09-14T03:43:18.633413+0000","last_clean":"2024-09-14T03:43:18.633413+0000","last_became_active":"2024-09-14T03:41:18.713287+0000","last_became_peered":"2024-09-14T03:41:18.713287+0000","last_unstale":"2024-09-14T03:43:18.633413+0000","last_undegraded":"2024-09-14T03:43:18.633413+0000","last_fullsized":"2024-09-14T03:43:18.633413+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:09.426148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545933+0000","last_change":"2024-09-14T03:42:35.827846+0000","last_active":"2024-09-14T03:43:17.545933+0000","last_peered":"2024-09-14T03:43:17.545933+0000","last_clean":"2024-09-14T03:43:17.545933+0000","last_became_active":"2024-09-14T03:42:35.827453+0000","last_became_peered":"2024-09-14T03:42:35.827453+0000","last_unstale":"2024-09-14T03:43:17.545933+0000","last_undegraded":"2024-09-14T03:43:17.545933+0000","last_fullsized":"2024-09-14T03:43:17.545933+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:00:40.709178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546772+0000","last_change":"2024-09-14T03:42:34.635824+0000","last_active":"2024-09-14T03:43:17.546772+0000","last_peered":"2024-09-14T03:43:17.546772+0000","last_clean":"2024-09-14T03:43:17.546772+0000","last_became_active":"2024-09-14T03:42:34.635531+0000","last_became_peered":"2024-09-14T03:42:34.635531+0000","last_unstale":"2024-09-14T03:43:17.546772+0000","last_undegraded":"2024-09-14T03:43:17.546772+0000","last_fullsized":"2024-09-14T03:43:17.546772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:08:15.718743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633970+0000","last_change":"2024-09-14T03:41:23.741514+0000","last_active":"2024-09-14T03:43:18.633970+0000","last_peered":"2024-09-14T03:43:18.633970+0000","last_clean":"2024-09-14T03:43:18.633970+0000","last_became_active":"2024-09-14T03:41:19.397559+0000","last_became_peered":"2024-09-14T03:41:19.397559+0000","last_unstale":"2024-09-14T03:43:18.633970+0000","last_undegraded":"2024-09-14T03:43:18.633970+0000","last_fullsized":"2024-09-14T03:43:18.633970+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:33:59.613404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148503+0000","last_change":"2024-09-14T03:42:35.827841+0000","last_active":"2024-09-14T03:43:18.148503+0000","last_peered":"2024-09-14T03:43:18.148503+0000","last_clean":"2024-09-14T03:43:18.148503+0000","last_became_active":"2024-09-14T03:42:35.827556+0000","last_became_peered":"2024-09-14T03:42:35.827556+0000","last_unstale":"2024-09-14T03:43:18.148503+0000","last_undegraded":"2024-09-14T03:43:18.148503+0000","last_fullsized":"2024-09-14T03:43:18.148503+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:27.869314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549947+0000","last_change":"2024-09-14T03:42:34.643876+0000","last_active":"2024-09-14T03:43:17.549947+0000","last_peered":"2024-09-14T03:43:17.549947+0000","last_clean":"2024-09-14T03:43:17.549947+0000","last_became_active":"2024-09-14T03:42:34.643603+0000","last_became_peered":"2024-09-14T03:42:34.643603+0000","last_unstale":"2024-09-14T03:43:17.549947+0000","last_undegraded":"2024-09-14T03:43:17.549947+0000","last_fullsized":"2024-09-14T03:43:17.549947+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:57.435387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.633784+0000","last_change":"2024-09-14T03:41:23.736591+0000","last_active":"2024-09-14T03:43:18.633784+0000","last_peered":"2024-09-14T03:43:18.633784+0000","last_clean":"2024-09-14T03:43:18.633784+0000","last_became_active":"2024-09-14T03:41:19.397811+0000","last_became_peered":"2024-09-14T03:41:19.397811+0000","last_unstale":"2024-09-14T03:43:18.633784+0000","last_undegraded":"2024-09-14T03:43:18.633784+0000","last_fullsized":"2024-09-14T03:43:18.633784+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:41.233137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040775400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548328+0000","last_change":"2024-09-14T03:42:35.667323+0000","last_active":"2024-09-14T03:43:17.548328+0000","last_peered":"2024-09-14T03:43:17.548328+0000","last_clean":"2024-09-14T03:43:17.548328+0000","last_became_active":"2024-09-14T03:42:35.667019+0000","last_became_peered":"2024-09-14T03:42:35.667019+0000","last_unstale":"2024-09-14T03:43:17.548328+0000","last_undegraded":"2024-09-14T03:43:17.548328+0000","last_fullsized":"2024-09-14T03:43:17.548328+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:02.864995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148637+0000","last_change":"2024-09-14T03:42:34.648053+0000","last_active":"2024-09-14T03:43:18.148637+0000","last_peered":"2024-09-14T03:43:18.148637+0000","last_clean":"2024-09-14T03:43:18.148637+0000","last_became_active":"2024-09-14T03:42:34.647766+0000","last_became_peered":"2024-09-14T03:42:34.647766+0000","last_unstale":"2024-09-14T03:43:18.148637+0000","last_undegraded":"2024-09-14T03:43:18.148637+0000","last_fullsized":"2024-09-14T03:43:18.148637+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:00:43.804911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.547073+0000","last_change":"2024-09-14T03:41:23.740297+0000","last_active":"2024-09-14T03:43:18.547073+0000","last_peered":"2024-09-14T03:43:18.547073+0000","last_clean":"2024-09-14T03:43:18.547073+0000","last_became_active":"2024-09-14T03:41:18.713102+0000","last_became_peered":"2024-09-14T03:41:18.713102+0000","last_unstale":"2024-09-14T03:43:18.547073+0000","last_undegraded":"2024-09-14T03:43:18.547073+0000","last_fullsized":"2024-09-14T03:43:18.547073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:37:04.215608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547411+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T03:43:17.547411+0000","last_peered":"2024-09-14T03:43:17.547411+0000","last_clean":"2024-09-14T03:43:17.547411+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T03:43:17.547411+0000","last_undegraded":"2024-09-14T03:43:17.547411+0000","last_fullsized":"2024-09-14T03:43:17.547411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.634965+0000","last_change":"2024-09-14T03:43:18.634965+0000","last_active":"2024-09-14T03:43:18.634965+0000","last_peered":"2024-09-14T03:43:18.634965+0000","last_clean":"2024-09-14T03:43:18.634965+0000","last_became_active":"2024-09-14T03:43:18.634451+0000","last_became_peered":"2024-09-14T03:43:18.634451+0000","last_unstale":"2024-09-14T03:43:18.634965+0000","last_undegraded":"2024-09-14T03:43:18.634965+0000","last_fullsized":"2024-09-14T03:43:18.634965+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:37.675709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547488+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T03:43:17.547488+0000","last_peered":"2024-09-14T03:43:17.547488+0000","last_clean":"2024-09-14T03:43:17.547488+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T03:43:17.547488+0000","last_undegraded":"2024-09-14T03:43:17.547488+0000","last_fullsized":"2024-09-14T03:43:17.547488+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545644+0000","last_change":"2024-09-14T03:42:35.664355+0000","last_active":"2024-09-14T03:43:17.545644+0000","last_peered":"2024-09-14T03:43:17.545644+0000","last_clean":"2024-09-14T03:43:17.545644+0000","last_became_active":"2024-09-14T03:42:35.664067+0000","last_became_peered":"2024-09-14T03:42:35.664067+0000","last_unstale":"2024-09-14T03:43:17.545644+0000","last_undegraded":"2024-09-14T03:43:17.545644+0000","last_fullsized":"2024-09-14T03:43:17.545644+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:19.133554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.551235+0000","last_change":"2024-09-14T03:42:34.644211+0000","last_active":"2024-09-14T03:43:17.551235+0000","last_peered":"2024-09-14T03:43:17.551235+0000","last_clean":"2024-09-14T03:43:17.551235+0000","last_became_active":"2024-09-14T03:42:34.643937+0000","last_became_peered":"2024-09-14T03:42:34.643937+0000","last_unstale":"2024-09-14T03:43:17.551235+0000","last_undegraded":"2024-09-14T03:43:17.551235+0000","last_fullsized":"2024-09-14T03:43:17.551235+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:34.755024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548840+0000","last_change":"2024-09-14T03:42:34.640845+0000","last_active":"2024-09-14T03:43:17.548840+0000","last_peered":"2024-09-14T03:43:17.548840+0000","last_clean":"2024-09-14T03:43:17.548840+0000","last_became_active":"2024-09-14T03:42:34.640500+0000","last_became_peered":"2024-09-14T03:42:34.640500+0000","last_unstale":"2024-09-14T03:43:17.548840+0000","last_undegraded":"2024-09-14T03:43:17.548840+0000","last_fullsized":"2024-09-14T03:43:17.548840+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:54.999114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548446+0000","last_change":"2024-09-14T03:42:35.662106+0000","last_active":"2024-09-14T03:43:17.548446+0000","last_peered":"2024-09-14T03:43:17.548446+0000","last_clean":"2024-09-14T03:43:17.548446+0000","last_became_active":"2024-09-14T03:42:35.661783+0000","last_became_peered":"2024-09-14T03:42:35.661783+0000","last_unstale":"2024-09-14T03:43:17.548446+0000","last_undegraded":"2024-09-14T03:43:17.548446+0000","last_fullsized":"2024-09-14T03:43:17.548446+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:42:03.398601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148835+0000","last_change":"2024-09-14T03:42:34.648183+0000","last_active":"2024-09-14T03:43:18.148835+0000","last_peered":"2024-09-14T03:43:18.148835+0000","last_clean":"2024-09-14T03:43:18.148835+0000","last_became_active":"2024-09-14T03:42:34.647898+0000","last_became_peered":"2024-09-14T03:42:34.647898+0000","last_unstale":"2024-09-14T03:43:18.148835+0000","last_undegraded":"2024-09-14T03:43:18.148835+0000","last_fullsized":"2024-09-14T03:43:18.148835+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:44.598862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549294+0000","last_change":"2024-09-14T03:42:35.664361+0000","last_active":"2024-09-14T03:43:17.549294+0000","last_peered":"2024-09-14T03:43:17.549294+0000","last_clean":"2024-09-14T03:43:17.549294+0000","last_became_active":"2024-09-14T03:42:35.664074+0000","last_became_peered":"2024-09-14T03:42:35.664074+0000","last_unstale":"2024-09-14T03:43:17.549294+0000","last_undegraded":"2024-09-14T03:43:17.549294+0000","last_fullsized":"2024-09-14T03:43:17.549294+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:16.964809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.550605+0000","last_change":"2024-09-14T03:42:35.827946+0000","last_active":"2024-09-14T03:43:17.550605+0000","last_peered":"2024-09-14T03:43:17.550605+0000","last_clean":"2024-09-14T03:43:17.550605+0000","last_became_active":"2024-09-14T03:42:35.827619+0000","last_became_peered":"2024-09-14T03:42:35.827619+0000","last_unstale":"2024-09-14T03:43:17.550605+0000","last_undegraded":"2024-09-14T03:43:17.550605+0000","last_fullsized":"2024-09-14T03:43:17.550605+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:35.250158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548436+0000","last_change":"2024-09-14T03:42:34.641678+0000","last_active":"2024-09-14T03:43:17.548436+0000","last_peered":"2024-09-14T03:43:17.548436+0000","last_clean":"2024-09-14T03:43:17.548436+0000","last_became_active":"2024-09-14T03:42:34.641420+0000","last_became_peered":"2024-09-14T03:42:34.641420+0000","last_unstale":"2024-09-14T03:43:17.548436+0000","last_undegraded":"2024-09-14T03:43:17.548436+0000","last_fullsized":"2024-09-14T03:43:17.548436+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:00.255578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148440+0000","last_change":"2024-09-14T03:42:35.665205+0000","last_active":"2024-09-14T03:43:18.148440+0000","last_peered":"2024-09-14T03:43:18.148440+0000","last_clean":"2024-09-14T03:43:18.148440+0000","last_became_active":"2024-09-14T03:42:35.664934+0000","last_became_peered":"2024-09-14T03:42:35.664934+0000","last_unstale":"2024-09-14T03:43:18.148440+0000","last_undegraded":"2024-09-14T03:43:18.148440+0000","last_fullsized":"2024-09-14T03:43:18.148440+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:13:21.255492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.545130+0000","last_change":"2024-09-14T03:42:34.650983+0000","last_active":"2024-09-14T03:43:17.545130+0000","last_peered":"2024-09-14T03:43:17.545130+0000","last_clean":"2024-09-14T03:43:17.545130+0000","last_became_active":"2024-09-14T03:42:34.650690+0000","last_became_peered":"2024-09-14T03:42:34.650690+0000","last_unstale":"2024-09-14T03:43:17.545130+0000","last_undegraded":"2024-09-14T03:43:17.545130+0000","last_fullsized":"2024-09-14T03:43:17.545130+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:59.946101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549518+0000","last_change":"2024-09-14T03:42:35.666729+0000","last_active":"2024-09-14T03:43:17.549518+0000","last_peered":"2024-09-14T03:43:17.549518+0000","last_clean":"2024-09-14T03:43:17.549518+0000","last_became_active":"2024-09-14T03:42:35.666452+0000","last_became_peered":"2024-09-14T03:42:35.666452+0000","last_unstale":"2024-09-14T03:43:17.549518+0000","last_undegraded":"2024-09-14T03:43:17.549518+0000","last_fullsized":"2024-09-14T03:43:17.549518+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:25.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548183+0000","last_change":"2024-09-14T03:42:34.651650+0000","last_active":"2024-09-14T03:43:17.548183+0000","last_peered":"2024-09-14T03:43:17.548183+0000","last_clean":"2024-09-14T03:43:17.548183+0000","last_became_active":"2024-09-14T03:42:34.651389+0000","last_became_peered":"2024-09-14T03:42:34.651389+0000","last_unstale":"2024-09-14T03:43:17.548183+0000","last_undegraded":"2024-09-14T03:43:17.548183+0000","last_fullsized":"2024-09-14T03:43:17.548183+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:28.695287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.546030+0000","last_change":"2024-09-14T03:42:34.663365+0000","last_active":"2024-09-14T03:43:17.546030+0000","last_peered":"2024-09-14T03:43:17.546030+0000","last_clean":"2024-09-14T03:43:17.546030+0000","last_became_active":"2024-09-14T03:42:34.663042+0000","last_became_peered":"2024-09-14T03:42:34.663042+0000","last_unstale":"2024-09-14T03:43:17.546030+0000","last_undegraded":"2024-09-14T03:43:17.546030+0000","last_fullsized":"2024-09-14T03:43:17.546030+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:29:06.456493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148585+0000","last_change":"2024-09-14T03:42:34.650367+0000","last_active":"2024-09-14T03:43:18.148585+0000","last_peered":"2024-09-14T03:43:18.148585+0000","last_clean":"2024-09-14T03:43:18.148585+0000","last_became_active":"2024-09-14T03:42:34.650086+0000","last_became_peered":"2024-09-14T03:42:34.650086+0000","last_unstale":"2024-09-14T03:43:18.148585+0000","last_undegraded":"2024-09-14T03:43:18.148585+0000","last_fullsized":"2024-09-14T03:43:18.148585+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:26.436325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547497+0000","last_change":"2024-09-14T03:42:35.664722+0000","last_active":"2024-09-14T03:43:17.547497+0000","last_peered":"2024-09-14T03:43:17.547497+0000","last_clean":"2024-09-14T03:43:17.547497+0000","last_became_active":"2024-09-14T03:42:35.664150+0000","last_became_peered":"2024-09-14T03:42:35.664150+0000","last_unstale":"2024-09-14T03:43:17.547497+0000","last_undegraded":"2024-09-14T03:43:17.547497+0000","last_fullsized":"2024-09-14T03:43:17.547497+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:33:31.543679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548706+0000","last_change":"2024-09-14T03:42:34.646474+0000","last_active":"2024-09-14T03:43:17.548706+0000","last_peered":"2024-09-14T03:43:17.548706+0000","last_clean":"2024-09-14T03:43:17.548706+0000","last_became_active":"2024-09-14T03:42:34.646139+0000","last_became_peered":"2024-09-14T03:42:34.646139+0000","last_unstale":"2024-09-14T03:43:17.548706+0000","last_undegraded":"2024-09-14T03:43:17.548706+0000","last_fullsized":"2024-09-14T03:43:17.548706+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:07.526545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547542+0000","last_change":"2024-09-14T03:42:35.661687+0000","last_active":"2024-09-14T03:43:17.547542+0000","last_peered":"2024-09-14T03:43:17.547542+0000","last_clean":"2024-09-14T03:43:17.547542+0000","last_became_active":"2024-09-14T03:42:35.661155+0000","last_became_peered":"2024-09-14T03:42:35.661155+0000","last_unstale":"2024-09-14T03:43:17.547542+0000","last_undegraded":"2024-09-14T03:43:17.547542+0000","last_fullsized":"2024-09-14T03:43:17.547542+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:16.038383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548715+0000","last_change":"2024-09-14T03:42:34.645745+0000","last_active":"2024-09-14T03:43:17.548715+0000","last_peered":"2024-09-14T03:43:17.548715+0000","last_clean":"2024-09-14T03:43:17.548715+0000","last_became_active":"2024-09-14T03:42:34.645455+0000","last_became_peered":"2024-09-14T03:42:34.645455+0000","last_unstale":"2024-09-14T03:43:17.548715+0000","last_undegraded":"2024-09-14T03:43:17.548715+0000","last_fullsized":"2024-09-14T03:43:17.548715+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:19.729438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.548642+0000","last_change":"2024-09-14T03:42:35.666516+0000","last_active":"2024-09-14T03:43:17.548642+0000","last_peered":"2024-09-14T03:43:17.548642+0000","last_clean":"2024-09-14T03:43:17.548642+0000","last_became_active":"2024-09-14T03:42:35.666172+0000","last_became_peered":"2024-09-14T03:42:35.666172+0000","last_unstale":"2024-09-14T03:43:17.548642+0000","last_undegraded":"2024-09-14T03:43:17.548642+0000","last_fullsized":"2024-09-14T03:43:17.548642+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:57:25.139258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.549760+0000","last_change":"2024-09-14T03:42:34.640794+0000","last_active":"2024-09-14T03:43:17.549760+0000","last_peered":"2024-09-14T03:43:17.549760+0000","last_clean":"2024-09-14T03:43:17.549760+0000","last_became_active":"2024-09-14T03:42:34.640417+0000","last_became_peered":"2024-09-14T03:42:34.640417+0000","last_unstale":"2024-09-14T03:43:17.549760+0000","last_undegraded":"2024-09-14T03:43:17.549760+0000","last_fullsized":"2024-09-14T03:43:17.549760+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:36.857651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:17.547109+0000","last_change":"2024-09-14T03:42:35.664713+0000","last_active":"2024-09-14T03:43:17.547109+0000","last_peered":"2024-09-14T03:43:17.547109+0000","last_clean":"2024-09-14T03:43:17.547109+0000","last_became_active":"2024-09-14T03:42:35.664138+0000","last_became_peered":"2024-09-14T03:42:35.664138+0000","last_unstale":"2024-09-14T03:43:17.547109+0000","last_undegraded":"2024-09-14T03:43:17.547109+0000","last_fullsized":"2024-09-14T03:43:17.547109+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:16:31.298143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:43:18.148751+0000","last_change":"2024-09-14T03:42:34.651887+0000","last_active":"2024-09-14T03:43:18.148751+0000","last_peered":"2024-09-14T03:43:18.148751+0000","last_clean":"2024-09-14T03:43:18.148751+0000","last_became_active":"2024-09-14T03:42:34.650986+0000","last_became_peered":"2024-09-14T03:42:34.650986+0000","last_unstale":"2024-09-14T03:43:18.148751+0000","last_undegraded":"2024-09-14T03:43:18.148751+0000","last_fullsized":"2024-09-14T03:43:18.148751+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:02:29.067192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28380,"kb_used_data":1356,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716772,"statfs":{"total":95995035648,"available":95965974528,"internally_reserved":0,"allocated":1388544,"data_stored":928583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.49199999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.223,"15min":0.223},"max":{"1min":0.66600000000000004,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.72799999999999998,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.56699999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.63500000000000001,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.33900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.68100000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72699999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.435,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.71999999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.70399999999999996}]},{"osd":4,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.47699999999999998,"15min":0.48499999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.5,"15min":0.503},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.52600000000000002,"15min":0.53200000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67400000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73699999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55200000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32456,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712696,"statfs":{"total":95995035648,"available":95961800704,"internally_reserved":0,"allocated":1372160,"data_stored":927902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54200000000000004,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.71999999999999997,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56599999999999995,"15min":0.57399999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77200000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72399999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.54500000000000004,"15min":0.55500000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78600000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70699999999999996,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76700000000000002,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.44600000000000001,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.437},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.46400000000000002,"15min":0.442},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72999999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.625}]},{"osd":5,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48399999999999999,"15min":0.46200000000000002},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73199999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.501,"15min":0.48499999999999999},"min":{"1min":0.32800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.53400000000000003,"15min":0.52200000000000002},"min":{"1min":0.39200000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53200000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":774144,"data_stored":336825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.35899999999999999,"5min":0.31,"15min":0.31},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.755,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.44400000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81499999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85299999999999998,"5min":0.879,"15min":0.879},"last":0.68400000000000005}]},{"osd":4,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.39600000000000002},"min":{"1min":0.25,"5min":0.246,"15min":0.246},"max":{"1min":0.71999999999999997,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.434},{"interface":"front","average":{"1min":0.442,"5min":0.41499999999999998,"15min":0.40200000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:42:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.47399999999999998,"15min":0.46500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.47299999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.876,"15min":0.876},"last":0.64700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27792,"kb_used_data":768,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717360,"statfs":{"total":95995035648,"available":95966576640,"internally_reserved":0,"allocated":786432,"data_stored":337872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.73999999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.52300000000000002,"15min":0.52500000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.67600000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.501}]},{"osd":1,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53800000000000003,"15min":0.54500000000000004},"min":{"1min":0.38200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77200000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.69099999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73699999999999999,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80300000000000005,"5min":0.87,"15min":0.87},"last":0.434},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.38500000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.67700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.41799999999999998,"15min":0.40200000000000002},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45300000000000001,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72799999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:43:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41699999999999998,"15min":0.39100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.71799999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.50600000000000001,"15min":0.47299999999999998},"min":{"1min":0.39300000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79100000000000004,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48099999999999998,"15min":0.47499999999999998},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.78200000000000003,"5min":1.008,"15min":1.008},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.48299999999999998,"15min":0.48099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.66800000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":770048,"data_stored":336525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.38700000000000001,"15min":0.39700000000000002},"min":{"1min":0.215,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51200000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.41099999999999998,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.43099999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.48599999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.439,"15min":0.44800000000000001},"min":{"1min":0.26200000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.64200000000000002,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.437}]},{"osd":2,"last update":"Sat Sep 14 03:43:18 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.46000000000000002,"15min":0.47799999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.54500000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41399999999999998,"15min":0.42199999999999999},"min":{"1min":0.29199999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.373,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.72799999999999998,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.39900000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72099999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.58599999999999997,"15min":0.57999999999999996},"min":{"1min":0.42199999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81499999999999995,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63600000000000001,"15min":0.64400000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74299999999999999,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.76500000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77400000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.64500000000000002,"15min":0.64900000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.67900000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31876,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713276,"statfs":{"total":95995035648,"available":95962394624,"internally_reserved":0,"allocated":774144,"data_stored":337051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38,"15min":0.379},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.47299999999999998,"5min":1.244,"15min":1.244},"last":0.318},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.373,"15min":0.37},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.58699999999999997,"5min":1.204,"15min":1.204},"last":0.371},{"interface":"front","average":{"1min":0.377,"5min":0.39000000000000001,"15min":0.39900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.53800000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41999999999999998,"15min":0.42299999999999999},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.42199999999999999,"15min":0.44500000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.50900000000000001,"5min":0.629,"15min":0.629},"last":0.441}]},{"osd":4,"last update":"Sat Sep 14 03:42:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.45300000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78900000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58299999999999996,"15min":0.57099999999999995},"min":{"1min":0.46600000000000003,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.76300000000000001,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.73999999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60199999999999998,"15min":0.61799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74299999999999999,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.629,"15min":0.63},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.80900000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.621,"15min":0.627},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73199999999999998,"5min":1.333,"15min":1.333},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.77300000000000002,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.68400000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":336454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.32700000000000001},"min":{"1min":0.20599999999999999,"5min":0.185,"15min":0.185},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.32900000000000001},"min":{"1min":0.26800000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36799999999999999,"15min":0.36099999999999999},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:43:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68700000000000006,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.66900000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 14 03:42:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59399999999999997,"15min":0.61199999999999999},"min":{"1min":0.436,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55700000000000005,"15min":0.56899999999999995},"min":{"1min":0.375,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Sep 14 03:43:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.622,"5min":0.63800000000000001,"15min":0.65400000000000003},"min":{"1min":0.42499999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.871,"15min":0.871},"last":0.81200000000000006}]},{"osd":6,"last update":"Sat Sep 14 03:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77600000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.72299999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.61099999999999999,"15min":0.61499999999999999},"min":{"1min":0.43099999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78900000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.88600000000000001,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.77900000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32456,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712696,"statfs":{"total":95995035648,"available":95961800704,"internally_reserved":0,"allocated":1368064,"data_stored":927407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.33400000000000002},"min":{"1min":0.28000000000000003,"5min":0.224,"15min":0.224},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.373,"15min":0.33200000000000002},"min":{"1min":0.27300000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.48899999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:43:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39100000000000001,"15min":0.378},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.55500000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.39300000000000002,"15min":0.371},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.57099999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.307}]},{"osd":3,"last update":"Sat Sep 14 03:43:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.44600000000000001,"15min":0.46800000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.52600000000000002,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.46300000000000002,"15min":0.48099999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68300000000000005,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.371}]},{"osd":4,"last update":"Sat Sep 14 03:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.752,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57599999999999996,"15min":0.56799999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.872,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84199999999999997,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.502}]},{"osd":6,"last update":"Sat Sep 14 03:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.65300000000000002,"15min":0.66100000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.88400000000000001,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.67000000000000004,"15min":0.68400000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:42:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58499999999999996,"15min":0.59099999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84499999999999997,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.64500000000000002,"15min":0.65300000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:43:36.340 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:43:36.340 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:43:36.340 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:43:36.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:43:36.550 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:36.845 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:36 smithi001 ceph-mon[29360]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:36.846 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:36 smithi001 ceph-mon[40972]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:36 smithi012 ceph-mon[33640]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:37 smithi001 ceph-mon[29360]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:37 smithi001 ceph-mon[40972]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:37 smithi012 ceph-mon[33640]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:43:38.502 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T03:43:38.503 INFO:teuthology.orchestra.run.smithi001.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:43:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:38 smithi012 ceph-mon[33640]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:38 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1330965661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:43:38.870 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:38 smithi001 ceph-mon[29360]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:38.870 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:38 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1330965661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:43:38.870 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:38 smithi001 ceph-mon[40972]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:38.870 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:38 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1330965661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:43:39.236 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:43:39.237 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:43:39.248 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-14T03:43:39.248 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T03:43:39.263 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T03:43:39.263 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:43:39.264 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-14T03:43:39.265 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T03:43:39.279 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T03:43:39.279 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-14T03:43:39.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-14T03:43:39.280 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:43:39.280 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-14T03:43:39.280 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi001.front.sepia.ceph.com 2024-09-14T03:43:39.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-14T03:43:39.507 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:39.512 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:40 smithi001 ceph-mon[29360]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:40 smithi001 ceph-mon[40972]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:40 smithi012 ceph-mon[33640]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:42.129 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:41] "GET /metrics HTTP/1.1" 200 33692 "" "Prometheus/2.43.0" 2024-09-14T03:43:42.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[29360]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:42.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/4087004133' entity='client.admin' 2024-09-14T03:43:42.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:42.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[40972]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:42.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/4087004133' entity='client.admin' 2024-09-14T03:43:42.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:42.715 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE Bus STOPPING 2024-09-14T03:43:42.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:43:42.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE Bus STOPPED 2024-09-14T03:43:42.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE Bus STARTING 2024-09-14T03:43:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:42 smithi012 ceph-mon[33640]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:42 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/4087004133' entity='client.admin' 2024-09-14T03:43:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:42 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:43.306 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE Serving on http://:::9283 2024-09-14T03:43:43.306 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:43:42] ENGINE Bus STARTED 2024-09-14T03:43:43.329 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-14T03:43:43.538 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:43.615 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_0' created 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:43.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:43 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:43 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-14T03:43:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:43 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:43 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:43 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:44.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[29360]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:44.824 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T03:43:44.824 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[29360]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:43:44.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[40972]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:44.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T03:43:44.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:44 smithi001 ceph-mon[40972]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:43:44.853 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-14T03:43:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:44 smithi012 ceph-mon[33640]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:44 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2434061919' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T03:43:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:44 smithi012 ceph-mon[33640]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:43:45.819 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:45.950 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:45 smithi001 ceph-mon[29360]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:43:45.950 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:45 smithi001 ceph-mon[40972]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:43:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:45 smithi012 ceph-mon[33640]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:43:46.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[40972]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[40972]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[40972]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[29360]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[29360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:43:46.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:46 smithi001 ceph-mon[29360]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:43:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:46 smithi012 ceph-mon[33640]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:46 smithi012 ceph-mon[33640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:43:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:46 smithi012 ceph-mon[33640]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:43:47.957 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[29360]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[40972]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:47 smithi012 ceph-mon[33640]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:43:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:43:48.729 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-14T03:43:49.085 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:48 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:43:49.085 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:48 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:43:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:48 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[40972]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[40972]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[29360]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:43:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:49 smithi001 ceph-mon[29360]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:43:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:49 smithi012 ceph-mon[33640]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:49 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1123396566' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:43:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:49 smithi012 ceph-mon[33640]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:43:50.217 INFO:teuthology.orchestra.run.smithi001.stdout:wait for mgr daemons to upgrade 2024-09-14T03:43:50.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:50 smithi001 ceph-mon[29360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:43:50.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:50 smithi001 ceph-mon[29360]: Cluster is now healthy 2024-09-14T03:43:50.926 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:50 smithi001 ceph-mon[40972]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:43:50.926 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:50 smithi001 ceph-mon[40972]: Cluster is now healthy 2024-09-14T03:43:50.950 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-14T03:43:51.025 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-14T03:43:51.028 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T03:43:51.028 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T03:43:51.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T03:43:51.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T03:43:51.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_86195 2024-09-14T03:43:51.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:43:51.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T03:43:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:50 smithi012 ceph-mon[33640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:43:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:50 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:43:51.227 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:51 smithi001 ceph-mon[29360]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:51 smithi001 ceph-mon[40972]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:51.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:43:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:43:51] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.43.0" 2024-09-14T03:43:52.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 4064 4048 252.986 253 0.00237286 0.00393357 2024-09-14T03:43:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:51 smithi012 ceph-mon[33640]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 235 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:43:53.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 8027 8011 250.303 247.688 0.00477025 0.00398832 2024-09-14T03:43:54.087 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 11375 11359 236.613 209.25 0.00315796 0.00421844 2024-09-14T03:43:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:53 smithi012 ceph-mon[33640]: pgmap v99: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2024-09-14T03:43:54.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:53 smithi001 ceph-mon[29360]: pgmap v99: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2024-09-14T03:43:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:53 smithi001 ceph-mon[40972]: pgmap v99: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2024-09-14T03:43:55.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 14623 14607 228.205 203 0.0034419 0.00437604 2024-09-14T03:43:55.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:54 smithi012 ceph-mon[33640]: from='client.24622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:54 smithi001 ceph-mon[29360]: from='client.24622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:54 smithi001 ceph-mon[40972]: from='client.24622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:56.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 17668 17652 220.624 190.312 0.00382152 0.00452556 2024-09-14T03:43:56.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:56 smithi001 ceph-mon[29360]: pgmap v100: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-09-14T03:43:56.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:56 smithi001 ceph-mon[40972]: pgmap v100: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-09-14T03:43:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:56 smithi012 ceph-mon[33640]: pgmap v100: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-09-14T03:43:57.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 20551 20535 213.882 180.188 0.00390299 0.00466974 2024-09-14T03:43:58.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 23470 23454 209.388 182.438 0.00584096 0.00477035 2024-09-14T03:43:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:43:58 smithi001 ceph-mon[29360]: pgmap v101: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s 2024-09-14T03:43:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:43:58 smithi001 ceph-mon[40972]: pgmap v101: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s 2024-09-14T03:43:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:43:58 smithi012 ceph-mon[33640]: pgmap v101: 89 pgs: 89 active+clean; 407 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s 2024-09-14T03:43:59.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 26130 26114 203.994 166.25 0.0514321 0.00488248 2024-09-14T03:44:00.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 28609 28593 198.543 154.938 0.00490368 0.00503129 2024-09-14T03:44:00.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:00 smithi001 ceph-mon[40972]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T03:44:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:00 smithi001 ceph-mon[29360]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T03:44:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:00 smithi012 ceph-mon[33640]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T03:44:01.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 31070 31054 194.069 153.812 0.00592591 0.00514749 2024-09-14T03:44:02.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 33364 33348 189.459 143.375 0.00648301 0.00527329 2024-09-14T03:44:02.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:01] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[29360]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[40972]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:02 smithi012 ceph-mon[33640]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:44:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:03.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 35744 35728 186.066 148.75 0.00663061 0.00536943 2024-09-14T03:44:04.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 38073 38057 182.949 145.562 0.0109308 0.00546048 2024-09-14T03:44:04.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:04 smithi001 ceph-mon[29360]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-14T03:44:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:04 smithi001 ceph-mon[40972]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-14T03:44:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:04 smithi012 ceph-mon[33640]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-14T03:44:05.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 40308 40292 179.858 139.688 0.00656405 0.00555439 2024-09-14T03:44:06.088 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 42534 42518 177.142 139.125 0.0118205 0.00564051 2024-09-14T03:44:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:06 smithi012 ceph-mon[33640]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:06.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:06 smithi001 ceph-mon[29360]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:06.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:06 smithi001 ceph-mon[40972]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:07.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 44662 44646 174.382 133 0.00652746 0.0057291 2024-09-14T03:44:08.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 46733 46717 171.738 129.438 0.00496856 0.00581783 2024-09-14T03:44:08.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:08 smithi001 ceph-mon[29360]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:08 smithi001 ceph-mon[40972]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:08 smithi012 ceph-mon[33640]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.36k op/s 2024-09-14T03:44:09.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 48768 48752 169.262 127.188 0.0143659 0.0059031 2024-09-14T03:44:10.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 50740 50724 166.84 123.25 0.00413059 0.00598822 2024-09-14T03:44:10.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:10 smithi001 ceph-mon[29360]: pgmap v107: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 205 MiB/s wr, 3.28k op/s 2024-09-14T03:44:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:10 smithi001 ceph-mon[40972]: pgmap v107: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 205 MiB/s wr, 3.28k op/s 2024-09-14T03:44:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:10 smithi012 ceph-mon[33640]: pgmap v107: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 205 MiB/s wr, 3.28k op/s 2024-09-14T03:44:11.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:44:11.088923+0000 min lat: 0.00117899 max lat: 0.0878558 avg lat: 0.00608538 2024-09-14T03:44:11.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:44:11.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 52565 52549 164.201 114.062 0.0121547 0.00608538 2024-09-14T03:44:12.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 54350 54334 161.694 111.562 0.0136419 0.00617937 2024-09-14T03:44:12.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:11] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-09-14T03:44:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:12 smithi001 ceph-mon[40972]: pgmap v108: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:12 smithi001 ceph-mon[29360]: pgmap v108: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:12 smithi012 ceph-mon[33640]: pgmap v108: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:13.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 56105 56089 159.33 109.688 0.0157669 0.00627167 2024-09-14T03:44:14.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 57698 57682 156.731 99.5625 0.00687312 0.00637562 2024-09-14T03:44:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:14 smithi001 ceph-mon[40972]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:14 smithi001 ceph-mon[29360]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:14 smithi012 ceph-mon[33640]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:15.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 59246 59230 154.231 96.75 0.0083185 0.00647933 2024-09-14T03:44:16.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 60957 60941 152.339 106.938 0.00496931 0.00655898 2024-09-14T03:44:16.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:16 smithi001 ceph-mon[40972]: pgmap v110: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:16.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:16 smithi001 ceph-mon[29360]: pgmap v110: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:16 smithi012 ceph-mon[33640]: pgmap v110: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:17.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 62686 62670 150.636 108.062 0.0175438 0.00663373 2024-09-14T03:44:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:17 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:17 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:17 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:18.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 64367 64351 148.947 105.062 0.0144822 0.00670873 2024-09-14T03:44:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:18 smithi001 ceph-mon[29360]: pgmap v111: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:18 smithi001 ceph-mon[40972]: pgmap v111: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:18 smithi012 ceph-mon[33640]: pgmap v111: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:44:19.089 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 66017 66001 147.31 103.125 0.00952236 0.00678354 2024-09-14T03:44:20.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 67674 67658 145.802 103.562 0.0136044 0.00685393 2024-09-14T03:44:20.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:20 smithi001 ceph-mon[29360]: pgmap v112: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:20 smithi001 ceph-mon[40972]: pgmap v112: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:20 smithi012 ceph-mon[33640]: pgmap v112: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-14T03:44:21.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 69295 69279 144.318 101.312 0.00492387 0.00692425 2024-09-14T03:44:22.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 70867 70851 142.832 98.25 0.0111477 0.00699634 2024-09-14T03:44:22.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:21] "GET /metrics HTTP/1.1" 200 36122 "" "Prometheus/2.43.0" 2024-09-14T03:44:22.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:22 smithi001 ceph-mon[29360]: pgmap v113: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:44:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:22 smithi001 ceph-mon[40972]: pgmap v113: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:44:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:22 smithi012 ceph-mon[33640]: pgmap v113: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:44:23.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 72377 72361 141.318 94.375 0.00675284 0.00707104 2024-09-14T03:44:24.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 73881 73865 139.883 94 0.00831029 0.00714379 2024-09-14T03:44:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:24 smithi001 ceph-mon[29360]: pgmap v114: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-14T03:44:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:24 smithi001 ceph-mon[40972]: pgmap v114: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-14T03:44:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:24 smithi012 ceph-mon[33640]: pgmap v114: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-14T03:44:25.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 75299 75283 138.376 88.625 0.0117209 0.00722159 2024-09-14T03:44:26.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 76760 76744 137.031 91.3125 0.00871699 0.00729234 2024-09-14T03:44:26.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:26 smithi001 ceph-mon[29360]: pgmap v115: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T03:44:26.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:26 smithi001 ceph-mon[40972]: pgmap v115: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T03:44:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:26 smithi012 ceph-mon[33640]: pgmap v115: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T03:44:27.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 78199 78183 135.722 89.9375 0.0153588 0.00736268 2024-09-14T03:44:28.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 15 79638 79623 134.486 90 0.0118469 0.00743094 2024-09-14T03:44:28.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:28 smithi001 ceph-mon[40972]: pgmap v116: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T03:44:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:28 smithi001 ceph-mon[29360]: pgmap v116: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T03:44:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:28 smithi012 ceph-mon[33640]: pgmap v116: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T03:44:29.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 81036 81020 133.245 87.3125 0.00604804 0.00749865 2024-09-14T03:44:30.090 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 82428 82412 132.059 87 0.0140435 0.0075675 2024-09-14T03:44:30.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:30 smithi001 ceph-mon[29360]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:30 smithi001 ceph-mon[40972]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:30 smithi012 ceph-mon[33640]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T03:44:31.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:44:31.090627+0000 min lat: 0.00117899 max lat: 0.0878558 avg lat: 0.00763431 2024-09-14T03:44:31.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:44:31.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 15 83802 83787 130.906 85.9375 0.0173751 0.00763431 2024-09-14T03:44:32.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 85187 85171 129.822 86.5 0.0114959 0.00769792 2024-09-14T03:44:32.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:31] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2024-09-14T03:44:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:32 smithi001 ceph-mon[29360]: pgmap v118: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:44:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:32 smithi001 ceph-mon[40972]: pgmap v118: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:44:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:32 smithi012 ceph-mon[33640]: pgmap v118: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:44:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:33.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 86540 86524 128.745 84.5625 0.020968 0.00776269 2024-09-14T03:44:34.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 87902 87886 127.73 85.125 0.00917947 0.00782395 2024-09-14T03:44:34.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:34 smithi001 ceph-mon[29360]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:44:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:34 smithi001 ceph-mon[40972]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:44:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:34 smithi012 ceph-mon[33640]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:44:35.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 89208 89192 126.682 81.625 0.00707105 0.00788829 2024-09-14T03:44:36.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 90538 90522 125.714 83.125 0.0160312 0.00794963 2024-09-14T03:44:36.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:36 smithi001 ceph-mon[40972]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:44:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:36 smithi001 ceph-mon[29360]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:44:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:36 smithi012 ceph-mon[33640]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:44:37.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 91849 91833 124.762 81.9375 0.0137534 0.00800997 2024-09-14T03:44:38.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 93111 93095 123.786 78.875 0.00776705 0.00807366 2024-09-14T03:44:38.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:38 smithi001 ceph-mon[29360]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:38.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:38 smithi001 ceph-mon[40972]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:38 smithi012 ceph-mon[33640]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:39.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 94357 94341 122.829 77.875 0.00947734 0.00813626 2024-09-14T03:44:40.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 95602 95586 121.91 77.8125 0.0198604 0.00819728 2024-09-14T03:44:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:40 smithi001 ceph-mon[29360]: pgmap v122: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T03:44:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:40 smithi001 ceph-mon[40972]: pgmap v122: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T03:44:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:40 smithi012 ceph-mon[33640]: pgmap v122: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T03:44:41.091 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 96825 96809 121.001 76.4375 0.00782447 0.00825926 2024-09-14T03:44:42.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 98038 98022 120.114 75.8125 0.00978887 0.00831919 2024-09-14T03:44:42.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:41] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2024-09-14T03:44:42.678 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:42 smithi001 ceph-mon[29360]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T03:44:42.678 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:42 smithi001 ceph-mon[40972]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T03:44:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:42 smithi012 ceph-mon[33640]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T03:44:43.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 99135 99119 119.122 68.5625 0.0147668 0.00838776 2024-09-14T03:44:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:43 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:43.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:43 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:43.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:43 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:44.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 100115 100099 118.03 61.25 0.0121827 0.00846728 2024-09-14T03:44:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:44 smithi012 ceph-mon[33640]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:44.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:44 smithi001 ceph-mon[29360]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:44 smithi001 ceph-mon[40972]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-14T03:44:45.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 101118 101102 117.005 62.6875 0.00507167 0.00853862 2024-09-14T03:44:46.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 102122 102106 116.019 62.75 0.0104633 0.00861403 2024-09-14T03:44:46.639 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:46 smithi012 ceph-mon[33640]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T03:44:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:46 smithi001 ceph-mon[29360]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T03:44:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:46 smithi001 ceph-mon[40972]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T03:44:47.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 103067 103051 115.002 59.0625 0.00994446 0.00869034 2024-09-14T03:44:47.539 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:47.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:44:48.093 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 104058 104042 114.07 61.9375 0.0150049 0.00876146 2024-09-14T03:44:48.544 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:48 smithi012 ceph-mon[33640]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T03:44:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:48 smithi001 ceph-mon[29360]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T03:44:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:48 smithi001 ceph-mon[40972]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T03:44:49.095 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 105110 105094 113.233 65.75 0.00736625 0.00882624 2024-09-14T03:44:50.095 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 106131 106115 112.396 63.8125 0.0130187 0.00889217 2024-09-14T03:44:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[29360]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T03:44:50.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:50.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:50.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[40972]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T03:44:50.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:50.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:49 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:49 smithi012 ceph-mon[33640]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T03:44:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:49 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:49 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:44:51.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:44:51.095518+0000 min lat: 0.00117899 max lat: 0.104105 avg lat: 0.00895457 2024-09-14T03:44:51.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:44:51.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 107181 107165 111.615 65.625 0.0156275 0.00895457 2024-09-14T03:44:52.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 108230 108214 110.86 65.5625 0.021291 0.0090157 2024-09-14T03:44:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:51 smithi001 ceph-mon[40972]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-14T03:44:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:51 smithi001 ceph-mon[29360]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-14T03:44:52.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:44:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:44:51] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-09-14T03:44:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:51 smithi012 ceph-mon[33640]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-14T03:44:53.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 15 109286 109271 110.136 66.0625 0.0192659 0.00907485 2024-09-14T03:44:54.096 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 110328 110312 109.42 65.0625 0.0187269 0.00913396 2024-09-14T03:44:54.147 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:53 smithi012 ceph-mon[33640]: pgmap v129: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-14T03:44:54.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:53 smithi001 ceph-mon[29360]: pgmap v129: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-14T03:44:54.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:53 smithi001 ceph-mon[40972]: pgmap v129: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-14T03:44:55.097 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 111263 111247 108.623 58.4375 0.0338773 0.00920107 2024-09-14T03:44:56.097 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 112213 112197 107.866 59.375 0.0138917 0.00926585 2024-09-14T03:44:56.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:55 smithi001 ceph-mon[40972]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T03:44:56.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:55 smithi001 ceph-mon[29360]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T03:44:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:55 smithi012 ceph-mon[33640]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T03:44:57.099 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 113209 113193 107.171 62.25 0.0115633 0.00932593 2024-09-14T03:44:58.102 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 114148 114132 106.443 58.6875 0.0126196 0.00938961 2024-09-14T03:44:58.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:57 smithi001 ceph-mon[40972]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:44:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:57 smithi001 ceph-mon[29360]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:44:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:57 smithi012 ceph-mon[33640]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:44:59.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 115227 115211 105.868 67.4375 0.0175414 0.00944073 2024-09-14T03:45:00.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 116291 116275 105.298 66.5 0.0138331 0.00949143 2024-09-14T03:45:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:44:59 smithi001 ceph-mon[29360]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T03:45:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:44:59 smithi001 ceph-mon[40972]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T03:45:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:44:59 smithi012 ceph-mon[33640]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T03:45:01.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 117303 117287 104.697 63.25 0.00704463 0.00954645 2024-09-14T03:45:01.186 INFO:teuthology.orchestra.run.smithi001.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:01 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:01.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:45:02.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 118319 118303 104.116 63.5 0.0157722 0.00959944 2024-09-14T03:45:02.183 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:01] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.43.0" 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:02.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:02.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:03.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 119363 119347 103.577 65.25 0.0120618 0.00964991 2024-09-14T03:45:04.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 120397 120381 103.043 64.625 0.0168115 0.00969996 2024-09-14T03:45:04.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:04 smithi001 ceph-mon[29360]: pgmap v134: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T03:45:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:04 smithi001 ceph-mon[40972]: pgmap v134: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T03:45:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:04 smithi012 ceph-mon[33640]: pgmap v134: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T03:45:05.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 121448 121432 102.538 65.6875 0.0222833 0.0097474 2024-09-14T03:45:06.103 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 122440 122424 101.998 62 0.0288016 0.00979859 2024-09-14T03:45:06.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:06 smithi001 ceph-mon[40972]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-14T03:45:06.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:06 smithi001 ceph-mon[29360]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-14T03:45:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:06 smithi012 ceph-mon[33640]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-14T03:45:07.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 123511 123495 101.536 66.9375 0.0078825 0.00984339 2024-09-14T03:45:08.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 124565 124549 101.073 65.875 0.0147188 0.00988891 2024-09-14T03:45:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:08 smithi001 ceph-mon[29360]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-14T03:45:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:08 smithi001 ceph-mon[40972]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-14T03:45:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:08 smithi012 ceph-mon[33640]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-14T03:45:08.612 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:09.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 125586 125570 100.595 63.8125 0.0117229 0.00993544 2024-09-14T03:45:10.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 126614 126598 100.135 64.25 0.0118357 0.00998151 2024-09-14T03:45:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:10 smithi001 ceph-mon[29360]: pgmap v137: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T03:45:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:10 smithi001 ceph-mon[40972]: pgmap v137: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T03:45:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:10 smithi012 ceph-mon[33640]: pgmap v137: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T03:45:11.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:45:11.103928+0000 min lat: 0.00117899 max lat: 0.104105 avg lat: 0.0100288 2024-09-14T03:45:11.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:45:11.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 127611 127595 99.6626 62.3125 0.0110074 0.0100288 2024-09-14T03:45:12.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 128644 128628 99.2293 64.5625 0.0183638 0.0100723 2024-09-14T03:45:12.216 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:11] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.43.0" 2024-09-14T03:45:12.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:12 smithi001 ceph-mon[29360]: pgmap v138: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T03:45:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:12 smithi001 ceph-mon[40972]: pgmap v138: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T03:45:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:12 smithi012 ceph-mon[33640]: pgmap v138: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T03:45:13.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 129591 129575 98.7409 59.1875 0.0148008 0.0101222 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: pgmap v139: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:45:14.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:13 smithi012 ceph-mon[33640]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi012 2024-09-14T03:45:14.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 130482 130466 98.2222 55.6875 0.0121067 0.0101761 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: pgmap v139: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[40972]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi012 2024-09-14T03:45:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: pgmap v139: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:45:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:13 smithi001 ceph-mon[29360]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi012 2024-09-14T03:45:15.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 131352 131336 97.7002 54.375 0.0316401 0.0102297 2024-09-14T03:45:16.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:15 smithi012 ceph-mon[33640]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:16.104 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 132243 132227 97.206 55.6875 0.0104269 0.0102823 2024-09-14T03:45:16.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:15 smithi001 ceph-mon[29360]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:16.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:15 smithi001 ceph-mon[40972]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:17.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:17.094 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:17.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 133128 133112 96.7182 55.3125 0.0230973 0.0103341 2024-09-14T03:45:17.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:17 smithi001 ceph-mon[29360]: pgmap v141: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T03:45:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:17 smithi001 ceph-mon[40972]: pgmap v141: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T03:45:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:17 smithi012 ceph-mon[33640]: pgmap v141: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T03:45:18.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 134013 133997 96.2423 55.3125 0.0345638 0.0103848 2024-09-14T03:45:18.149 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:45:19.070 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (2m) 18s ago 3m 29.0M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (3m) 30s ago 3m 89.5M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (2m) 18s ago 2m 17.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (2m) 30s ago 2m 12.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,8765 running (7m) 30s ago 7m 439M - 18.2.4-845-ge4e227b3 67ddd33fa72f 22ebe6faf5c2 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:9283,8765 running (9m) 18s ago 9m 489M - 18.2.4-845-ge4e227b3 67ddd33fa72f 05ee15394456 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (9m) 18s ago 9m 48.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (8m) 30s ago 8m 39.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (7m) 18s ago 7m 41.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (3m) 18s ago 3m 15.5M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:45:19.071 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (3m) 30s ago 3m 13.5M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (7m) 18s ago 7m 285M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (6m) 18s ago 6m 190M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (6m) 18s ago 6m 241M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (6m) 18s ago 6m 138M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (5m) 30s ago 5m 286M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (5m) 30s ago 5m 308M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (5m) 30s ago 5m 168M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (4m) 30s ago 4m 201M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:45:19.072 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (2m) 30s ago 4m 44.7M - 2.43.0 a07b618ecd1d 1d5996fb94fa 2024-09-14T03:45:19.072 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:18 smithi001 ceph-mon[40972]: from='client.24628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.073 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:18 smithi001 ceph-mon[40972]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.073 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:18 smithi001 ceph-mon[29360]: from='client.24628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.073 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:18 smithi001 ceph-mon[29360]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:18 smithi012 ceph-mon[33640]: from='client.24628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:18 smithi012 ceph-mon[33640]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 134868 134852 95.7559 53.4375 0.019991 0.0104378 2024-09-14T03:45:19.624 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:45:19.624 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:45:19.624 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:45:19.624 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 15 2024-09-14T03:45:19.625 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:45:19.626 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[40972]: pgmap v142: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[40972]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1134030004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[29360]: pgmap v142: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[29360]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:19 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1134030004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "0/2 daemons upgraded", 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 image on host smithi012", 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:45:20.091 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:45:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:19 smithi012 ceph-mon[33640]: pgmap v142: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T03:45:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:19 smithi012 ceph-mon[33640]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:19 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1134030004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:20.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 135736 135720 95.2896 54.25 0.0063765 0.0104895 2024-09-14T03:45:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:20 smithi012 ceph-mon[33640]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:21.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0063 2024-09-14T03:45:21.105 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 136629 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 94.8746 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 44.1398 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 253 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 53.4375 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1517 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 706.237 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 4048 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 855 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.010536 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.00695831 2024-09-14T03:45:21.106 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 0.104105 2024-09-14T03:45:21.107 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00117899 2024-09-14T03:45:21.110 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T03:45:21.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:20 smithi001 ceph-mon[29360]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:20 smithi001 ceph-mon[40972]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:21 smithi012 ceph-mon[33640]: pgmap v143: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2024-09-14T03:45:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:21 smithi001 ceph-mon[29360]: pgmap v143: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2024-09-14T03:45:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:21 smithi001 ceph-mon[40972]: pgmap v143: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2024-09-14T03:45:22.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:21] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-09-14T03:45:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:23 smithi012 ceph-mon[33640]: pgmap v144: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-14T03:45:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:23 smithi001 ceph-mon[29360]: pgmap v144: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-14T03:45:24.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:23 smithi001 ceph-mon[40972]: pgmap v144: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-14T03:45:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:25 smithi012 ceph-mon[33640]: pgmap v145: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.11k op/s 2024-09-14T03:45:26.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:25 smithi001 ceph-mon[40972]: pgmap v145: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.11k op/s 2024-09-14T03:45:26.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:25 smithi001 ceph-mon[29360]: pgmap v145: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.11k op/s 2024-09-14T03:45:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:27 smithi012 ceph-mon[33640]: pgmap v146: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 736 op/s 2024-09-14T03:45:28.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:27 smithi001 ceph-mon[29360]: pgmap v146: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 736 op/s 2024-09-14T03:45:28.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:27 smithi001 ceph-mon[40972]: pgmap v146: 89 pgs: 89 active+clean; 8.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 736 op/s 2024-09-14T03:45:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:29 smithi012 ceph-mon[33640]: pgmap v147: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.11k op/s 2024-09-14T03:45:30.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:29 smithi001 ceph-mon[29360]: pgmap v147: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.11k op/s 2024-09-14T03:45:30.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:29 smithi001 ceph-mon[40972]: pgmap v147: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.11k op/s 2024-09-14T03:45:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:31 smithi012 ceph-mon[33640]: pgmap v148: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 738 op/s 2024-09-14T03:45:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:31 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:32.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:31 smithi001 ceph-mon[29360]: pgmap v148: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 738 op/s 2024-09-14T03:45:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:31 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:32.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:31 smithi001 ceph-mon[40972]: pgmap v148: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 738 op/s 2024-09-14T03:45:32.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:31 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:32.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:31] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.43.0" 2024-09-14T03:45:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:33 smithi012 ceph-mon[33640]: pgmap v149: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 758 op/s 2024-09-14T03:45:34.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:33 smithi001 ceph-mon[40972]: pgmap v149: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 758 op/s 2024-09-14T03:45:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:33 smithi001 ceph-mon[29360]: pgmap v149: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 758 op/s 2024-09-14T03:45:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:35 smithi012 ceph-mon[33640]: pgmap v150: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.08k op/s 2024-09-14T03:45:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:35 smithi001 ceph-mon[29360]: pgmap v150: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.08k op/s 2024-09-14T03:45:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:35 smithi001 ceph-mon[40972]: pgmap v150: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.08k op/s 2024-09-14T03:45:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:37 smithi001 ceph-mon[29360]: pgmap v151: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:45:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:37 smithi001 ceph-mon[40972]: pgmap v151: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:45:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:37 smithi012 ceph-mon[33640]: pgmap v151: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:45:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:39 smithi001 ceph-mon[29360]: pgmap v152: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:45:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:39 smithi001 ceph-mon[40972]: pgmap v152: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:45:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:39 smithi012 ceph-mon[33640]: pgmap v152: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:45:42.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:41 smithi001 ceph-mon[29360]: pgmap v153: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T03:45:42.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:41 smithi001 ceph-mon[40972]: pgmap v153: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T03:45:42.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:41] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.43.0" 2024-09-14T03:45:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:41 smithi012 ceph-mon[33640]: pgmap v153: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T03:45:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:43 smithi001 ceph-mon[29360]: pgmap v154: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-14T03:45:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:43 smithi001 ceph-mon[40972]: pgmap v154: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-14T03:45:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:43 smithi012 ceph-mon[33640]: pgmap v154: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-14T03:45:46.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:46 smithi001 ceph-mon[29360]: pgmap v155: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:46.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:46 smithi001 ceph-mon[40972]: pgmap v155: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:46 smithi012 ceph-mon[33640]: pgmap v155: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:47 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:47 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:47 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:48 smithi001 ceph-mon[29360]: pgmap v156: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-14T03:45:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:48 smithi001 ceph-mon[40972]: pgmap v156: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-14T03:45:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:48 smithi012 ceph-mon[33640]: pgmap v156: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-14T03:45:50.560 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:45:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:50 smithi001 ceph-mon[40972]: pgmap v157: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:50 smithi001 ceph-mon[29360]: pgmap v157: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:50 smithi012 ceph-mon[33640]: pgmap v157: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:45:51.434 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:45:51.434 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (3m) 51s ago 4m 29.0M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:45:51.434 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (4m) 62s ago 4m 89.5M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:45:51.434 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (3m) 51s ago 3m 17.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (3m) 62s ago 3m 12.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,8765 running (8m) 62s ago 8m 439M - 18.2.4-845-ge4e227b3 67ddd33fa72f 22ebe6faf5c2 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:9283,8765 running (9m) 51s ago 9m 489M - 18.2.4-845-ge4e227b3 67ddd33fa72f 05ee15394456 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (9m) 51s ago 10m 48.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (8m) 62s ago 8m 39.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (8m) 51s ago 8m 41.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (3m) 51s ago 3m 15.5M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (3m) 62s ago 3m 13.5M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (7m) 51s ago 7m 285M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:45:51.435 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (7m) 51s ago 7m 190M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (7m) 51s ago 7m 241M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (6m) 51s ago 6m 138M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (6m) 62s ago 6m 286M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (5m) 62s ago 5m 308M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (5m) 62s ago 5m 168M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (5m) 62s ago 5m 201M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:45:51.436 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (3m) 62s ago 4m 44.7M - 2.43.0 a07b618ecd1d 1d5996fb94fa 2024-09-14T03:45:51.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:51 smithi001 ceph-mon[29360]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:51 smithi001 ceph-mon[40972]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:51 smithi012 ceph-mon[33640]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:51.979 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:45:51.979 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:45:51.980 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 15 2024-09-14T03:45:51.981 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:45:51.981 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:45:52.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:45:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:45:51] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.43.0" 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "0/2 daemons upgraded", 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 image on host smithi012", 2024-09-14T03:45:52.412 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:45:52.413 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[29360]: pgmap v158: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[29360]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[29360]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2891014348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[40972]: pgmap v158: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[40972]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[40972]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:52 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2891014348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:52 smithi012 ceph-mon[33640]: pgmap v158: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-14T03:45:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:52 smithi012 ceph-mon[33640]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:52 smithi012 ceph-mon[33640]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:52 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2891014348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:45:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:53 smithi001 ceph-mon[29360]: from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:53.725 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:53 smithi001 ceph-mon[40972]: from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:53 smithi012 ceph-mon[33640]: from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:45:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:54 smithi001 ceph-mon[29360]: pgmap v159: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:45:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:54 smithi001 ceph-mon[40972]: pgmap v159: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:45:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:54 smithi012 ceph-mon[33640]: pgmap v159: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:45:56.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:56 smithi001 ceph-mon[29360]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:45:56.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:56 smithi001 ceph-mon[40972]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:45:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:56 smithi012 ceph-mon[33640]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:45:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:45:58 smithi001 ceph-mon[29360]: pgmap v161: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-14T03:45:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:45:58 smithi001 ceph-mon[40972]: pgmap v161: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-14T03:45:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:45:58 smithi012 ceph-mon[33640]: pgmap v161: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-14T03:46:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:00 smithi001 ceph-mon[29360]: pgmap v162: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-14T03:46:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:00 smithi001 ceph-mon[40972]: pgmap v162: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-14T03:46:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:00 smithi012 ceph-mon[33640]: pgmap v162: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-14T03:46:02.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:01] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.43.0" 2024-09-14T03:46:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:02 smithi012 ceph-mon[33640]: pgmap v163: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-14T03:46:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:02 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:02.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:02 smithi001 ceph-mon[29360]: pgmap v163: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-14T03:46:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:02 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:02 smithi001 ceph-mon[40972]: pgmap v163: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-14T03:46:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:02 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: Upgrade: Updating mgr.x 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[29360]: Deploying daemon mgr.x on smithi012 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: Upgrade: Updating mgr.x 2024-09-14T03:46:03.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:46:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:46:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:03 smithi001 ceph-mon[40972]: Deploying daemon mgr.x on smithi012 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: Upgrade: Updating mgr.x 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:03 smithi012 ceph-mon[33640]: Deploying daemon mgr.x on smithi012 2024-09-14T03:46:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:04 smithi001 ceph-mon[29360]: pgmap v164: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-14T03:46:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:04 smithi001 ceph-mon[40972]: pgmap v164: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-14T03:46:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:04 smithi012 ceph-mon[33640]: pgmap v164: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-14T03:46:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:06 smithi001 ceph-mon[40972]: pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:46:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:06 smithi001 ceph-mon[29360]: pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:46:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:06 smithi012 ceph-mon[33640]: pgmap v165: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:46:08.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:07 smithi012 ceph-mon[33640]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:46:08.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:07 smithi001 ceph-mon[29360]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:46:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:07 smithi001 ceph-mon[40972]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:46:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:09 smithi012 ceph-mon[33640]: pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:10.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:09 smithi001 ceph-mon[40972]: pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:09 smithi001 ceph-mon[29360]: pgmap v167: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:11 smithi012 ceph-mon[33640]: pgmap v168: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:11 smithi001 ceph-mon[29360]: pgmap v168: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:11 smithi001 ceph-mon[40972]: pgmap v168: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:12.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:11] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.43.0" 2024-09-14T03:46:13.780 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:13 smithi012 systemd[1]: Stopping Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:13.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:13 smithi001 ceph-mon[29360]: pgmap v169: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:13 smithi001 ceph-mon[40972]: pgmap v169: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:13 smithi012 ceph-mon[33640]: pgmap v169: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T03:46:15.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:14 smithi012 podman[70257]: 2024-09-14 03:46:14.823347832 +0000 UTC m=+1.054326912 container died 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b) 2024-09-14T03:46:16.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:15 smithi012 ceph-mon[33640]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:46:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:15 smithi001 ceph-mon[29360]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:46:16.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:15 smithi001 ceph-mon[40972]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:46:16.492 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 podman[70257]: 2024-09-14 03:46:16.197305435 +0000 UTC m=+2.428284510 container cleanup 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:46:16.492 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 bash[70257]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:46:16.746 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 podman[70272]: 2024-09-14 03:46:16.491398452 +0000 UTC m=+1.657661350 container remove 22ebe6faf5c2e52fbeae873b8c91b1d7f971760a81c2ae6a99ca9a2dfb6f5c3c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T03:46:16.747 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:46:17.087 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:46:17.087 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 systemd[1]: Stopped Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:17.087 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:16 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Consumed 11.735s CPU time. 2024-09-14T03:46:17.088 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:16 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:17.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:16 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:17.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:16 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:17.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 systemd[1]: Starting Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:17.790 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 podman[70389]: 2024-09-14 03:46:17.369972951 +0000 UTC m=+0.029746800 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:46:17.790 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 podman[70389]: 2024-09-14 03:46:17.527012706 +0000 UTC m=+0.186786523 container create 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.license=GPLv2) 2024-09-14T03:46:18.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 podman[70389]: 2024-09-14 03:46:17.838887603 +0000 UTC m=+0.498661419 container init 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-14T03:46:18.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 podman[70389]: 2024-09-14 03:46:17.846767436 +0000 UTC m=+0.506541253 container start 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-a278231) 2024-09-14T03:46:18.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 bash[70389]: 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 2024-09-14T03:46:18.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:17 smithi012 systemd[1]: Started Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:18.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:17 smithi012 ceph-mon[33640]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-09-14T03:46:18.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:17 smithi001 ceph-mon[29360]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-09-14T03:46:18.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:17 smithi001 ceph-mon[40972]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:19 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:19 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:20 smithi012 ceph-mon[33640]: pgmap v172: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:46:20.415 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:20 smithi001 ceph-mon[29360]: pgmap v172: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:46:20.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:20 smithi001 ceph-mon[40972]: pgmap v172: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:46:22.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:21] "GET /metrics HTTP/1.1" 200 36123 "" "Prometheus/2.43.0" 2024-09-14T03:46:22.609 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:46:22.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-09-14T03:46:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:22 smithi012 ceph-mon[33640]: pgmap v173: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:22 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:22 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.880 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[29360]: pgmap v173: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:22.880 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.880 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.880 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[40972]: pgmap v173: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:22.881 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.881 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:22 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:22.881 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:46:23.751 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:46:23.751 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (3m) 2s ago 4m 27.5M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:46:23.751 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (4m) 94s ago 4m 89.5M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:46:23.751 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (3m) 2s ago 3m 18.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:46:23.751 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (3m) 94s ago 3m 12.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 starting - - - - 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:9283,8765 running (10m) 2s ago 10m 492M - 18.2.4-845-ge4e227b3 67ddd33fa72f 05ee15394456 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (10m) 2s ago 10m 49.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (9m) 94s ago 9m 39.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (8m) 2s ago 8m 39.8M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (4m) 2s ago 4m 15.7M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (4m) 94s ago 4m 13.5M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (8m) 2s ago 8m 381M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (8m) 2s ago 8m 249M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (7m) 2s ago 7m 318M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (7m) 2s ago 7m 175M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:46:23.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (6m) 94s ago 6m 286M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:46:23.753 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (6m) 94s ago 6m 308M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:46:23.753 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (6m) 94s ago 6m 168M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:46:23.753 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (5m) 94s ago 5m 201M 4017M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:46:23.753 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (3m) 94s ago 5m 44.7M - 2.43.0 a07b618ecd1d 1d5996fb94fa 2024-09-14T03:46:24.269 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:46:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:46:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:46:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 15 2024-09-14T03:46:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:46:24.271 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "1/2 daemons upgraded", 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:46:24.696 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:46:24.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: pgmap v174: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:24.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:24.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: from='client.24794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:24.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/601377401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: pgmap v174: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: from='client.24794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:24.948 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:24 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/601377401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: pgmap v174: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: from='client.24794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:24 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/601377401' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:25.827 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:25 smithi012 ceph-mon[33640]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.828 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:25 smithi012 ceph-mon[33640]: from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.828 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:25 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.828 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:25 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[29360]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[29360]: from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[40972]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[40972]: from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:25 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:26.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:26 smithi001 ceph-mon[29360]: pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:46:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:26 smithi001 ceph-mon[40972]: pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:46:26.991 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:26 smithi012 ceph-mon[33640]: pgmap v175: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:26 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:26] ENGINE Bus STARTING 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:26 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: CherryPy Checker: 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:26 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: The Application mounted at '' has an empty config. 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:26 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:27] ENGINE Serving on http://:::9283 2024-09-14T03:46:27.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:27] ENGINE Bus STARTED 2024-09-14T03:46:27.897 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: Standby manager daemon x restarted 2024-09-14T03:46:27.897 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:46:27.897 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:46:27.898 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:27.898 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:46:27.898 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:27 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: Standby manager daemon x restarted 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: Standby manager daemon x started 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: Standby manager daemon x restarted 2024-09-14T03:46:27.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:46:27.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:46:27.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:27.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:46:27.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:27 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:28.928 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:28 smithi012 ceph-mon[33640]: pgmap v176: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-14T03:46:28.929 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:28 smithi012 ceph-mon[33640]: mgrmap e21: y(active, since 5m), standbys: x 2024-09-14T03:46:28.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:28 smithi001 ceph-mon[29360]: pgmap v176: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-14T03:46:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:28 smithi001 ceph-mon[29360]: mgrmap e21: y(active, since 5m), standbys: x 2024-09-14T03:46:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:28 smithi001 ceph-mon[40972]: pgmap v176: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-14T03:46:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:28 smithi001 ceph-mon[40972]: mgrmap e21: y(active, since 5m), standbys: x 2024-09-14T03:46:29.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:46:29 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:46:29.493Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2024-09-14T03:46:30.953 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:30 smithi012 ceph-mon[33640]: pgmap v177: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T03:46:30.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:30 smithi001 ceph-mon[29360]: pgmap v177: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T03:46:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:30 smithi001 ceph-mon[40972]: pgmap v177: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T03:46:32.156 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:31] "GET /metrics HTTP/1.1" 200 36123 "" "Prometheus/2.43.0" 2024-09-14T03:46:32.156 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:31.884+0000 7fe434464640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:46:32.156 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setuser ceph since I am not root 2024-09-14T03:46:32.157 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ignoring --setgroup ceph since I am not root 2024-09-14T03:46:32.157 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:32.151+0000 7f0023a81040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:46:32.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:31 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:31] ENGINE Bus STOPPING 2024-09-14T03:46:32.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:46:32.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:32] ENGINE Bus STOPPED 2024-09-14T03:46:32.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:32.246+0000 7f0023a81040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:32.820 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:32.475+0000 7f0023a81040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:46:32.820 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:32.578+0000 7f0023a81040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.821 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: Activating manager daemon x 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: mgrmap e22: x(active, starting, since 0.0226644s) 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:46:32.822 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[29360]: Manager daemon x is now available 2024-09-14T03:46:32.823 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: Activating manager daemon x 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: mgrmap e22: x(active, starting, since 0.0226644s) 2024-09-14T03:46:32.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:46:32.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:46:32.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:32.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:32.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:32.826 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-mon[40972]: Manager daemon x is now available 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: pgmap v178: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:46:33.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: Activating manager daemon x 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.24350 172.21.15.1:0/2408297108' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: mgrmap e22: x(active, starting, since 0.0226644s) 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:46:33.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:33.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:32 smithi012 ceph-mon[33640]: Manager daemon x is now available 2024-09-14T03:46:33.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:32.818+0000 7f0023a81040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:33.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:33.037+0000 7f0023a81040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:46:33.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:33.279Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:46:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:33.279Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:46:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:33.281Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:46:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:33.282Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:46:33.597 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:33.282Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:46:33.895 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:33.541+0000 7f0023a81040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:46:33.896 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:33.636+0000 7f0023a81040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:46:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-mon[29360]: mgrmap e23: x(active, since 1.01934s) 2024-09-14T03:46:34.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-mon[40972]: mgrmap e23: x(active, since 1.01934s) 2024-09-14T03:46:34.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:33.927+0000 7f0023a81040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:46:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:33 smithi012 ceph-mon[33640]: mgrmap e23: x(active, since 1.01934s) 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:34] ENGINE Bus STARTING 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:34] ENGINE Serving on http://:::9283 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:46:34] ENGINE Bus STARTED 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: pgmap v3: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:35.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:34 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: pgmap v3: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: pgmap v3: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:35.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:35.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:35.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:35.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:34 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.101+0000 7f2b2ee3b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.101+0000 7f2b2ee3b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.101+0000 7f2b2ee3b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.101+0000 7f2b2ee3b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.101+0000 7f2b2ee3b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.425 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.104+0000 7f2b268ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.426 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.104+0000 7f2b268ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.426 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.104+0000 7f2b268ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.426 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.104+0000 7f2b268ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.426 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:35.104+0000 7f2b268ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:46:35.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:35.298+0000 7f0023a81040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:46:35.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:35.394+0000 7f0023a81040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[29360]: Checking for cert/key for grafana.a 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[29360]: Migrating grafana.a cert to cert store 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[29360]: Migrating grafana.a key to cert store 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[40972]: Checking for cert/key for grafana.a 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[40972]: Migrating grafana.a cert to cert store 2024-09-14T03:46:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[40972]: Migrating grafana.a key to cert store 2024-09-14T03:46:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:35 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-mon[33640]: Checking for cert/key for grafana.a 2024-09-14T03:46:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-mon[33640]: Migrating grafana.a cert to cert store 2024-09-14T03:46:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-mon[33640]: Migrating grafana.a key to cert store 2024-09-14T03:46:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:35 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:46:36.861 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:36.469+0000 7f0023a81040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:46:36.862 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:36.645+0000 7f0023a81040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[40972]: Deploying cephadm binary to smithi001 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[40972]: Deploying cephadm binary to smithi012 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[40972]: mgrmap e24: x(active, since 4s) 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[29360]: Deploying cephadm binary to smithi001 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[29360]: Deploying cephadm binary to smithi012 2024-09-14T03:46:37.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-mon[29360]: mgrmap e24: x(active, since 4s) 2024-09-14T03:46:37.180 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:36.860+0000 7f0023a81040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:37.180 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:36.956+0000 7f0023a81040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:46:37.180 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.177+0000 7f0023a81040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:46:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:36 smithi012 ceph-mon[33640]: Deploying cephadm binary to smithi001 2024-09-14T03:46:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:36 smithi012 ceph-mon[33640]: Deploying cephadm binary to smithi012 2024-09-14T03:46:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:36 smithi012 ceph-mon[33640]: mgrmap e24: x(active, since 4s) 2024-09-14T03:46:37.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.277+0000 7f0023a81040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:46:37.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.374+0000 7f0023a81040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:46:37.912 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.619+0000 7f0023a81040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:46:37.912 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.714+0000 7f0023a81040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:46:37.912 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.819+0000 7f0023a81040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:46:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-mon[40972]: pgmap v5: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-mon[29360]: pgmap v5: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:38.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:37.963+0000 7f0023a81040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:38.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:38.051+0000 7f0023a81040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:46:38.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:38.145+0000 7f0023a81040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:46:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-mon[33640]: pgmap v5: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:38.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:37.871+0000 7f2b0fb80640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:46:37] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:38.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: Traceback (most recent call last): 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: self._connections.run(self.expiration_interval) 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: self._run(expiration_interval) 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: s = self.context.wrap_socket( 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: return self.sslsocket_class._create( 2024-09-14T03:46:38.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:38.348 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: self.do_handshake() 2024-09-14T03:46:38.348 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:38.348 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: self._sslobj.do_handshake() 2024-09-14T03:46:38.348 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:38.348 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:46:37 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:46:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[29360]: [14/Sep/2024:03:46:37] ENGINE Bus STARTING 2024-09-14T03:46:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[29360]: [14/Sep/2024:03:46:37] ENGINE Serving on https://172.21.15.12:7150 2024-09-14T03:46:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[29360]: [14/Sep/2024:03:46:37] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:39.222 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[29360]: mgrmap e25: x(active, since 6s) 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[40972]: [14/Sep/2024:03:46:37] ENGINE Bus STARTING 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[40972]: [14/Sep/2024:03:46:37] ENGINE Serving on https://172.21.15.12:7150 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[40972]: [14/Sep/2024:03:46:37] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:39.223 INFO:journalctl@ceph.mon.c.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:39.224 INFO:journalctl@ceph.mon.c.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: self.do_handshake() 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:39.225 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:38 smithi001 ceph-mon[40972]: mgrmap e25: x(active, since 6s) 2024-09-14T03:46:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:38 smithi012 ceph-mon[33640]: [14/Sep/2024:03:46:37] ENGINE Bus STARTING 2024-09-14T03:46:39.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:38 smithi012 ceph-mon[33640]: [14/Sep/2024:03:46:37] ENGINE Serving on https://172.21.15.12:7150 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:38 smithi012 ceph-mon[33640]: [14/Sep/2024:03:46:37] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: Traceback (most recent call last): 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._run(expiration_interval) 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:39.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: s = self.context.wrap_socket( 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: return self.sslsocket_class._create( 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: self.do_handshake() 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._sslobj.do_handshake() 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:39.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:38 smithi012 ceph-mon[33640]: mgrmap e25: x(active, since 6s) 2024-09-14T03:46:39.917 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:39.496+0000 7f0023a81040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:46:39.917 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:46:39] ENGINE Bus STARTING 2024-09-14T03:46:39.917 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: CherryPy Checker: 2024-09-14T03:46:39.917 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: The Application mounted at '' has an empty config. 2024-09-14T03:46:39.918 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: 2024-09-14T03:46:39.918 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:46:39] ENGINE Serving on http://:::9283 2024-09-14T03:46:39.918 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: [14/Sep/2024:03:46:39] ENGINE Bus STARTED 2024-09-14T03:46:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: pgmap v6: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: [14/Sep/2024:03:46:37] ENGINE Serving on http://172.21.15.12:8765 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: [14/Sep/2024:03:46:37] ENGINE Bus STARTED 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: Standby manager daemon y started 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: pgmap v6: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: [14/Sep/2024:03:46:37] ENGINE Serving on http://172.21.15.12:8765 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: [14/Sep/2024:03:46:37] ENGINE Bus STARTED 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: Standby manager daemon y started 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:46:40.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:39 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: pgmap v6: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-09-14T03:46:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: [14/Sep/2024:03:46:37] ENGINE Serving on http://172.21.15.12:8765 2024-09-14T03:46:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: [14/Sep/2024:03:46:37] ENGINE Bus STARTED 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: Standby manager daemon y started 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:46:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:39 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/1642144077' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:46:41.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:41 smithi012 ceph-mon[33640]: pgmap v7: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 538 op/s 2024-09-14T03:46:41.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:41 smithi012 ceph-mon[33640]: mgrmap e26: x(active, since 8s), standbys: y 2024-09-14T03:46:41.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:41 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[29360]: pgmap v7: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 538 op/s 2024-09-14T03:46:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[29360]: mgrmap e26: x(active, since 8s), standbys: y 2024-09-14T03:46:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[40972]: pgmap v7: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 538 op/s 2024-09-14T03:46:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[40972]: mgrmap e26: x(active, since 8s), standbys: y 2024-09-14T03:46:41.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:42.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:43 smithi012 ceph-mon[33640]: pgmap v8: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 418 op/s 2024-09-14T03:46:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:43 smithi001 ceph-mon[40972]: pgmap v8: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 418 op/s 2024-09-14T03:46:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:43 smithi001 ceph-mon[29360]: pgmap v8: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 418 op/s 2024-09-14T03:46:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:44 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:44 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:44 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:44 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:44 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:44 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: pgmap v9: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi001 to 2481M 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: pgmap v9: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi001 to 2481M 2024-09-14T03:46:45.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:45 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: pgmap v9: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-14T03:46:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi001 to 2481M 2024-09-14T03:46:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:45 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:46 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:46 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:46 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:46 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:46 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:46.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:46 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:47.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:47 smithi012 ceph-mon[33640]: pgmap v10: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-14T03:46:47.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:47 smithi001 ceph-mon[29360]: pgmap v10: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-14T03:46:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:47 smithi001 ceph-mon[40972]: pgmap v10: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-14T03:46:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:49 smithi012 ceph-mon[33640]: pgmap v11: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 677 op/s 2024-09-14T03:46:49.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:49 smithi001 ceph-mon[29360]: pgmap v11: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 677 op/s 2024-09-14T03:46:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:49 smithi001 ceph-mon[40972]: pgmap v11: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 677 op/s 2024-09-14T03:46:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:50 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:50 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:50 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: pgmap v12: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.01k op/s 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: Adjusting osd_memory_target on smithi012 to 2481M 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: pgmap v12: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.01k op/s 2024-09-14T03:46:51.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: Adjusting osd_memory_target on smithi012 to 2481M 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:51.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: pgmap v12: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.01k op/s 2024-09-14T03:46:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: Adjusting osd_memory_target on smithi012 to 2481M 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:51.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:51 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:46:52.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:46:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[29611]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:46:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[40972]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:52.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:52 smithi001 ceph-mon[29360]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:46:52.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:52.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:46:52.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:52.837 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:52 smithi012 ceph-mon[33640]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:46:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:53 smithi012 ceph-mon[33640]: pgmap v13: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-14T03:46:53.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:53 smithi001 ceph-mon[29360]: pgmap v13: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-14T03:46:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:53 smithi001 ceph-mon[40972]: pgmap v13: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-14T03:46:55.255 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:46:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:55 smithi012 ceph-mon[33640]: pgmap v14: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:55 smithi001 ceph-mon[29360]: pgmap v14: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:55 smithi001 ceph-mon[40972]: pgmap v14: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (4m) 13s ago 5m 27.5M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (5m) 12s ago 5m 89.7M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (4m) 13s ago 4m 18.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (4m) 12s ago 4m 13.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:46:56.290 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (38s) 12s ago 9m 500M - 19.3.0-4907-ga2782315 8801d2529e43 9495d35485e2 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:9283,8765 running (10m) 13s ago 10m 432M - 18.2.4-845-ge4e227b3 67ddd33fa72f 05ee15394456 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (11m) 13s ago 11m 54.9M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (9m) 12s ago 9m 40.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (9m) 13s ago 9m 46.7M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (4m) 13s ago 4m 15.4M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (4m) 12s ago 4m 15.9M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (8m) 13s ago 8m 385M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (8m) 13s ago 8m 253M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (8m) 13s ago 8m 321M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (7m) 13s ago 7m 176M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (7m) 12s ago 7m 385M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (7m) 12s ago 7m 419M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:46:56.291 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (6m) 12s ago 6m 218M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:46:56.292 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (6m) 12s ago 6m 263M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:46:56.292 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (4m) 12s ago 5m 52.0M - 2.43.0 a07b618ecd1d 1d5996fb94fa 2024-09-14T03:46:56.346 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 systemd[1]: Stopping Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:56.623 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-mon[33640]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.623 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-mon[33640]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 1, 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:56.901 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 14, 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:46:56.902 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:46:56.902 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:56 smithi001 ceph-mon[40972]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.902 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:56 smithi001 ceph-mon[40972]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.903 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:56 smithi001 ceph-mon[29360]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.903 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:56 smithi001 ceph-mon[29360]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:56.930 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.621Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:46:56.930 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.621Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.621Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.621Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.622Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.622Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.622Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.622Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.622Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.623Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.623Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[66797]: ts=2024-09-14T03:46:56.623Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:46:56.931 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:56 smithi012 podman[77713]: 2024-09-14 03:46:56.632215079 +0000 UTC m=+0.152887944 container died 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:57.279 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 podman[77713]: 2024-09-14 03:46:57.026421487 +0000 UTC m=+0.547094363 container cleanup 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:57.279 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 bash[77713]: ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "1/2 daemons upgraded", 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "", 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:46:57.341 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:46:57.563 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 podman[77728]: 2024-09-14 03:46:57.336629546 +0000 UTC m=+0.691544710 container remove 1d5996fb94faf7194ad38067d520b1e9af7572574b632a12cbad3682157213d2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[29360]: pgmap v15: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[29360]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2219895135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[40972]: pgmap v15: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[40972]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:57 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2219895135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:57.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:57 smithi012 ceph-mon[33640]: pgmap v15: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:57.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:57 smithi012 ceph-mon[33640]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:57.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:57 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2219895135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:46:57.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:46:57.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 systemd[1]: Stopped Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:57.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.399s CPU time. 2024-09-14T03:46:57.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 systemd[1]: Starting Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:58.309 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 podman[77819]: 2024-09-14 03:46:57.867018777 +0000 UTC m=+0.037399494 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:46:58.309 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:57 smithi012 podman[77819]: 2024-09-14 03:46:57.988089694 +0000 UTC m=+0.158470386 container create 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:58.309 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 podman[77819]: 2024-09-14 03:46:58.307816481 +0000 UTC m=+0.478197187 container init 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 podman[77819]: 2024-09-14 03:46:58.317491771 +0000 UTC m=+0.487872465 container start 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.372Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.372Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.372Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi012 (none))" 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.372Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.372Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.375Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:46:58.572 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.377Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.383Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.383Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.386Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.386Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.093µs 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.386Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.388Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.410Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.411Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.411Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.712µs wal_replay_duration=25.003344ms wbl_replay_duration=154ns total_replay_duration=25.12173ms 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.414Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.414Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:46:58.573 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.414Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:46:58.574 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 bash[77819]: 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff 2024-09-14T03:46:58.574 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 systemd[1]: Started Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:58.574 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.457Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=42.525284ms db_storage=1.425µs remote_storage=1.313µs web_handler=529ns query_engine=930ns scrape=1.037033ms scrape_sd=205.094µs notify=23.133µs notify_sd=15.263µs rules=40.444252ms tracing=8.286µs 2024-09-14T03:46:58.574 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.459Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:46:58.574 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:46:58.459Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:46:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-mon[33640]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:58 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:58.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[29360]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:58.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:58.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[40972]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:58.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:46:58.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:58 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[29360]: pgmap v16: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:59.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[29360]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:46:59.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[29360]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:46:59.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[40972]: pgmap v16: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:59.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:59.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:59.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[40972]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:46:59.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:46:59 smithi001 ceph-mon[40972]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:46:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:59 smithi012 ceph-mon[33640]: pgmap v16: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:46:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:59 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:59 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:59.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:59 smithi012 ceph-mon[33640]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:46:59.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:46:59 smithi012 ceph-mon[33640]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: pgmap v17: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: Upgrade: Updating mgr.y 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[40972]: Deploying daemon mgr.y on smithi001 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: pgmap v17: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: Upgrade: Updating mgr.y 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:01.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:01 smithi001 ceph-mon[29360]: Deploying daemon mgr.y on smithi001 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: pgmap v17: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: Upgrade: Updating mgr.y 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:01.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:01 smithi012 ceph-mon[33640]: Deploying daemon mgr.y on smithi001 2024-09-14T03:47:02.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:02 smithi001 systemd[1]: Stopping Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:03.356 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:03 smithi001 podman[90120]: 2024-09-14 03:47:03.050805455 +0000 UTC m=+0.180446177 container died 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:47:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:03 smithi001 ceph-mon[29360]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:47:03.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:03 smithi001 podman[90120]: 2024-09-14 03:47:03.563415135 +0000 UTC m=+0.693055852 container cleanup 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:47:03.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:03 smithi001 bash[90120]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y 2024-09-14T03:47:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:03 smithi001 ceph-mon[40972]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:47:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:03 smithi012 ceph-mon[33640]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:47:03.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:47:03] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2024-09-14T03:47:04.082 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:03 smithi001 podman[90135]: 2024-09-14 03:47:03.824108437 +0000 UTC m=+0.764210064 container remove 05ee153944568d41480d32d87b06cf1f2b1f6d85b9dea82341f14d64bb240c43 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-14T03:47:04.082 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:03 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:47:04.398 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-14T03:47:04.398 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 systemd[1]: Stopped Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:04.398 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service: Consumed 52.468s CPU time. 2024-09-14T03:47:04.719 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 systemd[1]: Starting Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:04.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 podman[90252]: 2024-09-14 03:47:04.625714126 +0000 UTC m=+0.037562886 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:47:04.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:04 smithi001 podman[90252]: 2024-09-14 03:47:04.733706449 +0000 UTC m=+0.145555199 container create 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=main-a278231) 2024-09-14T03:47:05.372 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:05 smithi001 podman[90252]: 2024-09-14 03:47:05.028640809 +0000 UTC m=+0.440489559 container init 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T03:47:05.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:05 smithi001 podman[90252]: 2024-09-14 03:47:05.042887322 +0000 UTC m=+0.454736073 container start 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-14T03:47:05.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:05 smithi001 bash[90252]: 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 2024-09-14T03:47:05.373 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:05 smithi001 systemd[1]: Started Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[40972]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:47:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[29360]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:47:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:05 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:05 smithi012 ceph-mon[33640]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:47:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:05 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:05 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:05 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:05 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:06 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:06 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:07.224 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:06 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:07.224 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:06 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:07.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:06 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:07.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:06 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:07.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:07 smithi012 ceph-mon[33640]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:07 smithi001 ceph-mon[40972]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:08.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:07 smithi001 ceph-mon[29360]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:09.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:09 smithi001 ceph-mon[29360]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:09.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:09 smithi001 ceph-mon[40972]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:10.092 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:09 smithi012 ceph-mon[33640]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-09-14T03:47:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:11 smithi012 ceph-mon[33640]: pgmap v22: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:11 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:11 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[29360]: pgmap v22: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:11.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:11.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[40972]: pgmap v22: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:11.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:11.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:11 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:12 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:12 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:12 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:12 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:12 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-mon[40972]: pgmap v23: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-14T03:47:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-mon[29360]: pgmap v23: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-14T03:47:13.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:13 smithi012 ceph-mon[33640]: pgmap v23: 89 pgs: 89 active+clean; 2.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-14T03:47:13.849 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:47:13] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-09-14T03:47:14.061 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:13] ENGINE Bus STARTING 2024-09-14T03:47:14.062 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: CherryPy Checker: 2024-09-14T03:47:14.062 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: The Application mounted at '' has an empty config. 2024-09-14T03:47:14.062 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:14.062 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:13] ENGINE Serving on http://:::9283 2024-09-14T03:47:14.062 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:13] ENGINE Bus STARTED 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: Standby manager daemon y restarted 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: Standby manager daemon y started 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:47:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: Standby manager daemon y restarted 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: Standby manager daemon y started 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:47:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:14 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: Standby manager daemon y restarted 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: Standby manager daemon y started 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:47:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:14 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:15 smithi001 ceph-mon[40972]: pgmap v24: 89 pgs: 89 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:15 smithi001 ceph-mon[40972]: mgrmap e27: x(active, since 42s), standbys: y 2024-09-14T03:47:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:15 smithi001 ceph-mon[29360]: pgmap v24: 89 pgs: 89 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:15 smithi001 ceph-mon[29360]: mgrmap e27: x(active, since 42s), standbys: y 2024-09-14T03:47:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:15 smithi012 ceph-mon[33640]: pgmap v24: 89 pgs: 89 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:15 smithi012 ceph-mon[33640]: mgrmap e27: x(active, since 42s), standbys: y 2024-09-14T03:47:16.199 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:16] ENGINE Bus STOPPING 2024-09-14T03:47:16.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:16.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:16] ENGINE Bus STOPPED 2024-09-14T03:47:16.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:47:16.152+0000 7f2bbd3b6640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:47:16.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ignoring --setuser ceph since I am not root 2024-09-14T03:47:16.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ignoring --setgroup ceph since I am not root 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: Activating manager daemon y 2024-09-14T03:47:17.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: mgrmap e28: y(active, starting, since 0.0113741s) 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:47:17.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:47:17.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:16 smithi012 ceph-mon[33640]: Manager daemon y is now available 2024-09-14T03:47:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: Activating manager daemon y 2024-09-14T03:47:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: mgrmap e28: y(active, starting, since 0.0113741s) 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:47:17.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[29360]: Manager daemon y is now available 2024-09-14T03:47:17.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: Activating manager daemon y 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.14883 172.21.15.12:0/1240334876' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:47:17.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: mgrmap e28: y(active, starting, since 0.0113741s) 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:47:17.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:47:17.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:16 smithi001 ceph-mon[40972]: Manager daemon y is now available 2024-09-14T03:47:17.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:17] ENGINE Bus STARTING 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.641+0000 7f327e3f5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.641+0000 7f327e3f5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.641+0000 7f327e3f5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.641+0000 7f327e3f5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.641+0000 7f327e3f5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.645+0000 7f3278029640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.645+0000 7f3278029640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.645+0000 7f3278029640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.645+0000 7f3278029640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:17.645+0000 7f3278029640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:47:17.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:17] ENGINE Serving on http://:::9283 2024-09-14T03:47:17.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:47:17] ENGINE Bus STARTED 2024-09-14T03:47:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:47:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: mgrmap e29: y(active, since 1.29808s) 2024-09-14T03:47:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: mgrmap e29: y(active, since 1.29808s) 2024-09-14T03:47:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: mgrmap e29: y(active, since 1.29808s) 2024-09-14T03:47:18.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:47:19.300 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:19.301 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[40972]: pgmap v3: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:19.301 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[40972]: mgrmap e30: y(active, since 2s) 2024-09-14T03:47:19.301 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:19.301 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[29360]: pgmap v3: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:19.301 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:18 smithi001 ceph-mon[29360]: mgrmap e30: y(active, since 2s) 2024-09-14T03:47:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:18 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:47:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:18 smithi012 ceph-mon[33640]: pgmap v3: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:18 smithi012 ceph-mon[33640]: mgrmap e30: y(active, since 2s) 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:19.421+0000 7f326133c640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:47:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: Traceback (most recent call last): 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._connections.run(self.expiration_interval) 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._run(expiration_interval) 2024-09-14T03:47:19.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: s = self.context.wrap_socket( 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: return self.sslsocket_class._create( 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self.do_handshake() 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._sslobj.do_handshake() 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:19.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:47:20.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-mon[29360]: pgmap v4: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:20.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:19 smithi001 ceph-mon[40972]: pgmap v4: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:19 smithi012 ceph-mon[33640]: pgmap v4: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: [14/Sep/2024:03:47:19] ENGINE Bus STARTING 2024-09-14T03:47:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: [14/Sep/2024:03:47:19] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:47:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: [14/Sep/2024:03:47:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: Traceback (most recent call last): 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._run(expiration_interval) 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: s = self.context.wrap_socket( 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: return self.sslsocket_class._create( 2024-09-14T03:47:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self.do_handshake() 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._sslobj.do_handshake() 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: [14/Sep/2024:03:47:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: [14/Sep/2024:03:47:19] ENGINE Bus STARTED 2024-09-14T03:47:21.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:21 smithi012 ceph-mon[33640]: mgrmap e31: y(active, since 3s) 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: [14/Sep/2024:03:47:19] ENGINE Bus STARTING 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: [14/Sep/2024:03:47:19] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: [14/Sep/2024:03:47:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:47:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:21.472 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: [14/Sep/2024:03:47:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: [14/Sep/2024:03:47:19] ENGINE Bus STARTED 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[29360]: mgrmap e31: y(active, since 3s) 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: [14/Sep/2024:03:47:19] ENGINE Bus STARTING 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: [14/Sep/2024:03:47:19] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: [14/Sep/2024:03:47:19] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:21.473 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:21.474 INFO:journalctl@ceph.mon.c.smithi001.stdout: self.do_handshake() 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: [14/Sep/2024:03:47:19] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: [14/Sep/2024:03:47:19] ENGINE Bus STARTED 2024-09-14T03:47:21.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:21 smithi001 ceph-mon[40972]: mgrmap e31: y(active, since 3s) 2024-09-14T03:47:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:22 smithi012 ceph-mon[33640]: pgmap v5: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:22.374 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:22 smithi001 ceph-mon[29360]: pgmap v5: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:22.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:22 smithi001 ceph-mon[40972]: pgmap v5: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[29360]: pgmap v6: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[40972]: pgmap v6: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:23 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:23 smithi012 ceph-mon[33640]: pgmap v6: 89 pgs: 89 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-09-14T03:47:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:23 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:23 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:23 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:23 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[40972]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 26 KiB/s rd, 308 B/s wr, 553 op/s 2024-09-14T03:47:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:25 smithi001 ceph-mon[29360]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 26 KiB/s rd, 308 B/s wr, 553 op/s 2024-09-14T03:47:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-mon[33640]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 26 KiB/s rd, 308 B/s wr, 553 op/s 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:47:25] ENGINE Bus STARTING 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: CherryPy Checker: 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: The Application mounted at '' has an empty config. 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:47:25] ENGINE Serving on http://:::9283 2024-09-14T03:47:25.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:25 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: [14/Sep/2024:03:47:25] ENGINE Bus STARTED 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: Standby manager daemon x started 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:26.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:26 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:26.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:26.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:47:26.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:26.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:26.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:26.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:26 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1470829799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[29360]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 236 B/s wr, 425 op/s 2024-09-14T03:47:27.562 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[29360]: mgrmap e32: y(active, since 10s), standbys: x 2024-09-14T03:47:27.563 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:47:27.563 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[40972]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 236 B/s wr, 425 op/s 2024-09-14T03:47:27.563 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[40972]: mgrmap e32: y(active, since 10s), standbys: x 2024-09-14T03:47:27.563 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:27 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:47:27.564 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:27 smithi012 ceph-mon[33640]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 236 B/s wr, 425 op/s 2024-09-14T03:47:27.564 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:27 smithi012 ceph-mon[33640]: mgrmap e32: y(active, since 10s), standbys: x 2024-09-14T03:47:27.564 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:27 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:47:27.911 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:47:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:28 smithi001 ceph-mon[40972]: from='client.24899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:28 smithi001 ceph-mon[29360]: from='client.24899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:28 smithi012 ceph-mon[33640]: from='client.24899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:28.921 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (5m) 5s ago 5m 27.5M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (5m) 4s ago 5m 89.2M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (4m) 5s ago 4m 19.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (4m) 4s ago 4m 13.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (71s) 4s ago 9m 139M - 19.3.0-4907-ga2782315 8801d2529e43 9495d35485e2 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (24s) 5s ago 11m 492M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (11m) 5s ago 11m 55.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (10m) 4s ago 10m 46.1M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (10m) 5s ago 10m 46.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (5m) 5s ago 5m 15.5M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (5m) 4s ago 5m 15.9M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:47:28.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (9m) 5s ago 9m 389M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (9m) 5s ago 9m 255M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (8m) 5s ago 8m 325M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (8m) 5s ago 8m 177M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (7m) 4s ago 7m 388M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (7m) 4s ago 7m 426M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (7m) 4s ago 7m 222M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (6m) 4s ago 6m 266M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:47:28.923 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (31s) 4s ago 6m 40.5M - 2.43.0 a07b618ecd1d 1948f1965d5f 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:47:29.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:47:29.590 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:47:30.184 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:47:30.184 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:47:30.184 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr" 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "", 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:47:30.185 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s rd, 192 B/s wr, 721 op/s 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='client.24790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:30.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:30.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1157434255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:30.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:30 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s rd, 192 B/s wr, 721 op/s 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='client.24790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1157434255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s rd, 192 B/s wr, 721 op/s 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='client.24790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1157434255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:30.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:30 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:47:31.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:31 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:31.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:31.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:31 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:32 smithi012 ceph-mon[33640]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:32 smithi012 ceph-mon[33640]: from='client.24796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:32 smithi012 ceph-mon[33640]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:47:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:32 smithi012 ceph-mon[33640]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[29360]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[29360]: from='client.24796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[29360]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[29360]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[40972]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[40972]: from='client.24796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[40972]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:47:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:32 smithi001 ceph-mon[40972]: Reconfiguring daemon prometheus.a on smithi012 2024-09-14T03:47:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:33.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:33.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[70401]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:34.243 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-mon[33640]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:34 smithi001 ceph-mon[29360]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:34 smithi001 ceph-mon[40972]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 640 op/s 2024-09-14T03:47:34.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 systemd[1]: Stopping Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:35.298 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.888Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:47:35.298 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.889Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:47:35.298 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:47:35.298 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.890Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.891Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.891Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.891Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[77832]: ts=2024-09-14T03:47:34.892Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:47:35.299 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:34 smithi012 podman[79778]: 2024-09-14 03:47:34.905209624 +0000 UTC m=+0.168635044 container died 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:35.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 podman[79778]: 2024-09-14 03:47:35.400852295 +0000 UTC m=+0.664277708 container cleanup 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:35.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 bash[79778]: ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:47:35.987 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 podman[79796]: 2024-09-14 03:47:35.735888543 +0000 UTC m=+0.825919403 container remove 1948f1965d5fcf5ce46d54f5668cc101fe49d6a2bc4e0277794fd13eb8196fff (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:35.988 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:47:35.988 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 systemd[1]: Stopped Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:35.988 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.621s CPU time. 2024-09-14T03:47:36.318 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-mon[33640]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 978 op/s 2024-09-14T03:47:36.319 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:35 smithi012 systemd[1]: Starting Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:36.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:36 smithi001 ceph-mon[29360]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 978 op/s 2024-09-14T03:47:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:36 smithi001 ceph-mon[40972]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 978 op/s 2024-09-14T03:47:36.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 podman[79880]: 2024-09-14 03:47:36.227138148 +0000 UTC m=+0.041120985 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:47:36.596 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 podman[79880]: 2024-09-14 03:47:36.362470528 +0000 UTC m=+0.176453331 container create 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:37.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 podman[79880]: 2024-09-14 03:47:36.665600544 +0000 UTC m=+0.479583359 container init 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:37.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 podman[79880]: 2024-09-14 03:47:36.677257366 +0000 UTC m=+0.491240174 container start 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.719Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.719Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.719Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi012 (none))" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.719Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.719Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.727Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.728Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.730Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.730Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.731Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:47:37.098 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.731Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.903µs 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.731Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.742Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.759Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.761Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 bash[79880]: 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 systemd[1]: Started Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.777Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.777Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.341µs wal_replay_duration=45.126151ms wbl_replay_duration=149ns total_replay_duration=45.260792ms 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.780Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.782Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.782Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:47:37.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.813Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=30.825391ms db_storage=1.436µs remote_storage=1.388µs web_handler=458ns query_engine=1.021µs scrape=507.219µs scrape_sd=167.763µs notify=16.63µs notify_sd=14.104µs rules=29.499495ms tracing=7.912µs 2024-09-14T03:47:37.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.813Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:47:37.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:47:36 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:47:36.813Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:47:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:47:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:37 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:47:38.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:37 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:38 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:38 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:38 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:38 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:39 smithi012 ceph-mon[33640]: pgmap v14: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:40.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:39 smithi012 ceph-mon[33640]: Upgrade: Updating mgr.x 2024-09-14T03:47:40.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:39 smithi012 ceph-mon[33640]: Deploying daemon mgr.x on smithi012 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[29360]: pgmap v14: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[29360]: Upgrade: Updating mgr.x 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[29360]: Deploying daemon mgr.x on smithi012 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[40972]: pgmap v14: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[40972]: Upgrade: Updating mgr.x 2024-09-14T03:47:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:39 smithi001 ceph-mon[40972]: Deploying daemon mgr.x on smithi012 2024-09-14T03:47:41.050 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:41 smithi012 systemd[1]: Stopping Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:41.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:41 smithi012 podman[80198]: 2024-09-14 03:47:41.566086572 +0000 UTC m=+0.190909638 container died 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:47:42.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:41 smithi001 ceph-mon[29360]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:42.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:41 smithi001 ceph-mon[40972]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:42.224 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:47:41] "GET /metrics HTTP/1.1" 200 36191 "" "Prometheus/2.43.0" 2024-09-14T03:47:42.256 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 podman[80198]: 2024-09-14 03:47:42.0097963 +0000 UTC m=+0.634619363 container cleanup 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:47:42.256 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 bash[80198]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:47:42.257 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:41 smithi012 ceph-mon[33640]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:42.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 podman[80214]: 2024-09-14 03:47:42.255064887 +0000 UTC m=+0.680007905 container remove 9495d35485e2d311d7aa2bcbf2b6a2ffd6935a5c51cab678d2ced78c7b5fc1e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:47:42.597 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:47:43.016 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:47:43.016 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 systemd[1]: Stopped Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:43.016 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:42 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service: Consumed 30.801s CPU time. 2024-09-14T03:47:43.344 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 systemd[1]: Starting Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:47:43.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 podman[80328]: 2024-09-14 03:47:43.24950098 +0000 UTC m=+0.029932576 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:47:43.596 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 podman[80328]: 2024-09-14 03:47:43.382533098 +0000 UTC m=+0.162964691 container create 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:47:44.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 podman[80328]: 2024-09-14 03:47:43.689089121 +0000 UTC m=+0.469520712 container init 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:47:44.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 podman[80328]: 2024-09-14 03:47:43.695319029 +0000 UTC m=+0.475750617 container start 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:47:44.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 bash[80328]: 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 2024-09-14T03:47:44.096 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:43 smithi012 systemd[1]: Started Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:47:44.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:43 smithi012 ceph-mon[33640]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:43 smithi001 ceph-mon[29360]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:43 smithi001 ceph-mon[40972]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:44 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:44 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:44 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:44 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:45 smithi012 ceph-mon[33640]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:46.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:45 smithi001 ceph-mon[29360]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:46.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:45 smithi001 ceph-mon[40972]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:47:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:47 smithi001 ceph-mon[40972]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-14T03:47:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:47 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:47 smithi001 ceph-mon[29360]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-14T03:47:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:47 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:48.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:47 smithi012 ceph-mon[33640]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-14T03:47:48.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:47 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:49 smithi001 ceph-mon[29360]: pgmap v19: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:47:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:49 smithi001 ceph-mon[40972]: pgmap v19: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:47:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:49 smithi012 ceph-mon[33640]: pgmap v19: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:47:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:51 smithi012 ceph-mon[33640]: pgmap v20: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:51 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:51 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:51.876 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[29360]: pgmap v20: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:51.876 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:51.876 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:51.876 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[40972]: pgmap v20: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:51.876 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:51.877 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:47:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:47:51] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.43.0" 2024-09-14T03:47:52.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:52 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:52 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:52 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:52 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:53.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:47:52] ENGINE Bus STARTING 2024-09-14T03:47:53.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: CherryPy Checker: 2024-09-14T03:47:53.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: The Application mounted at '' has an empty config. 2024-09-14T03:47:53.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: 2024-09-14T03:47:53.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:47:53] ENGINE Serving on http://:::9283 2024-09-14T03:47:53.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:47:53] ENGINE Bus STARTED 2024-09-14T03:47:53.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: pgmap v21: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:53.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: Standby manager daemon x restarted 2024-09-14T03:47:53.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:47:53.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:53.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:53.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:53.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:53 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: pgmap v21: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: Standby manager daemon x restarted 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: Standby manager daemon x started 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[29360]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: pgmap v21: 89 pgs: 89 active+clean; 1017 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: Standby manager daemon x restarted 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:47:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:53 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/1357077406' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:47:54.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:54 smithi012 ceph-mon[33640]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-14T03:47:54.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:54 smithi001 ceph-mon[29360]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-14T03:47:54.925 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:54 smithi001 ceph-mon[40972]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-14T03:47:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:55 smithi012 ceph-mon[33640]: pgmap v22: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-09-14T03:47:55.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:55 smithi001 ceph-mon[29360]: pgmap v22: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-09-14T03:47:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:55 smithi001 ceph-mon[40972]: pgmap v22: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-09-14T03:47:58.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: pgmap v23: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:47:58.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all mgr 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:47:58.100 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.101 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.102 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.102 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.102 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:57 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.115 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: pgmap v23: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all mgr 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:47:58.116 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.117 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: pgmap v23: 89 pgs: 89 active+clean; 784 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.118 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all mgr 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:47:58.120 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:57 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all crash 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all rgw 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all nfs 2024-09-14T03:47:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all crash 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all rgw 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all nfs 2024-09-14T03:47:59.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:58 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all crash 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all rgw 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:47:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:47:59.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all nfs 2024-09-14T03:47:59.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:58 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[29360]: pgmap v24: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[29360]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[29360]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[40972]: pgmap v24: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[40972]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:48:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:47:59 smithi001 ceph-mon[40972]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:48:00.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:47:59 smithi001 systemd[1]: Stopping Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:59 smithi012 ceph-mon[33640]: pgmap v24: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-09-14T03:48:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:59 smithi012 ceph-mon[33640]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:48:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:47:59 smithi012 ceph-mon[33640]: Deploying daemon node-exporter.a on smithi001 2024-09-14T03:48:00.713 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:00 smithi001 podman[97852]: 2024-09-14 03:48:00.233958522 +0000 UTC m=+0.164855678 container died d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:00.713 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:48:01.414 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 podman[97852]: 2024-09-14 03:48:01.11091234 +0000 UTC m=+1.041809504 container cleanup d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:01.414 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 bash[97852]: ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T03:48:01.672 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 podman[97870]: 2024-09-14 03:48:01.413098686 +0000 UTC m=+1.180927684 container remove d5fcc4e47b62e6e69a32e8152a42bf9fb1c9c82ce03f7f1f60d5eadf1cded3ba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:01.672 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (5m) 38s ago 6m 27.5M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (6m) 11s ago 6m 89.7M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (5m) 38s ago 5m 19.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (5m) 11s ago 5m 13.9M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (18s) 11s ago 10m 65.4M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (57s) 38s ago 12m 492M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (12m) 38s ago 12m 55.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (10m) 11s ago 10m 43.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:48:01.759 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (10m) 38s ago 10m 46.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (6m) 38s ago 6m 15.5M - 1.5.0 0da6a335fe13 d5fcc4e47b62 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (5m) 11s ago 5m 15.6M - 1.5.0 0da6a335fe13 9b45854423bb 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (10m) 38s ago 10m 389M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (9m) 38s ago 9m 255M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (9m) 38s ago 9m 325M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (8m) 38s ago 8m 177M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (8m) 11s ago 8m 391M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (8m) 11s ago 8m 428M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (7m) 11s ago 7m 225M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (7m) 11s ago 7m 271M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:48:01.760 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (25s) 11s ago 6m 38.0M - 2.43.0 a07b618ecd1d 53ce995a2d51 2024-09-14T03:48:01.938 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:01 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:01] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.43.0" 2024-09-14T03:48:01.938 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T03:48:01.938 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 systemd[1]: Stopped Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:01.938 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:01 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.073s CPU time. 2024-09-14T03:48:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:01 smithi001 ceph-mon[29360]: pgmap v25: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:01 smithi001 ceph-mon[29360]: from='client.24808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:01 smithi001 ceph-mon[40972]: pgmap v25: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:01 smithi001 ceph-mon[40972]: from='client.24808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:02.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 systemd[1]: Starting Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:48:02.335 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:48:02.336 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:48:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:01 smithi012 ceph-mon[33640]: pgmap v25: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:01 smithi012 ceph-mon[33640]: from='client.24808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:02.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 bash[98078]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:48:02.877 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:48:02.877 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:48:02.877 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr" 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:48:02.878 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[29360]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[29360]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/668618577' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[40972]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[40972]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:02 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/668618577' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:03.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 bash[98078]: Getting image source signatures 2024-09-14T03:48:03.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 bash[98078]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:48:03.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 bash[98078]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:48:03.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:02 smithi001 bash[98078]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:48:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:02 smithi012 ceph-mon[33640]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:02 smithi012 ceph-mon[33640]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:02 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:02 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/668618577' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:03 smithi001 ceph-mon[40972]: pgmap v26: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:03 smithi001 ceph-mon[40972]: from='client.24947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:03 smithi001 ceph-mon[29360]: pgmap v26: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:03 smithi001 ceph-mon[29360]: from='client.24947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:03 smithi012 ceph-mon[33640]: pgmap v26: 89 pgs: 89 active+clean; 521 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-14T03:48:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:03 smithi012 ceph-mon[33640]: from='client.24947 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:05.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:04 smithi001 bash[98078]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:48:05.221 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 bash[98078]: Writing manifest to image destination 2024-09-14T03:48:05.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 podman[98078]: 2024-09-14 03:48:05.321437661 +0000 UTC m=+3.072701637 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:48:05.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 podman[98078]: 2024-09-14 03:48:05.445102568 +0000 UTC m=+3.196366540 container create 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[29360]: pgmap v27: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[40972]: pgmap v27: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 podman[98078]: 2024-09-14 03:48:05.73150101 +0000 UTC m=+3.482764993 container init 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 podman[98078]: 2024-09-14 03:48:05.741554522 +0000 UTC m=+3.492818496 container start 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.748Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.748Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.750Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.750Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.751Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.751Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:48:06.222 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:48:06.223 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.752Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:48:06.224 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:48:06.225 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.753Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.754Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a[98167]: ts=2024-09-14T03:48:05.754Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:48:06.226 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 bash[98078]: 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 2024-09-14T03:48:06.227 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 03:48:05 smithi001 systemd[1]: Started Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:05 smithi012 ceph-mon[33640]: pgmap v27: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-09-14T03:48:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:05 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:05 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:07.586 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 136629 objects 2024-09-14T03:48:07.587 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :166.476 2024-09-14T03:48:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:07 smithi001 ceph-mon[29360]: pgmap v28: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:48:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:07 smithi001 ceph-mon[40972]: pgmap v28: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:48:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:07 smithi012 ceph-mon[33640]: pgmap v28: 89 pgs: 89 active+clean; 273 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:48:08.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:08 smithi012 systemd[1]: Stopping Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:08.712 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:08 smithi012 podman[83708]: 2024-09-14 03:48:08.448460489 +0000 UTC m=+0.141408003 container died 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:08.965 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:08 smithi012 podman[83708]: 2024-09-14 03:48:08.811114175 +0000 UTC m=+0.504061694 container cleanup 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:08.965 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:08 smithi012 bash[83708]: ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T03:48:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:08 smithi001 ceph-mon[29360]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:48:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:08 smithi001 ceph-mon[29360]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:48:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:08 smithi001 ceph-mon[40972]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:48:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:08 smithi001 ceph-mon[40972]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:48:09.240 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 podman[83724]: 2024-09-14 03:48:09.112477566 +0000 UTC m=+0.662865000 container remove 9b45854423bbce11ff2e86b77fa2630a870124a38a7fece0b3a02c0afc1b786c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:09.240 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:48:09.240 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:08 smithi012 ceph-mon[33640]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:48:09.240 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:08 smithi012 ceph-mon[33640]: Deploying daemon node-exporter.b on smithi012 2024-09-14T03:48:09.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T03:48:09.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 systemd[1]: Stopped Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:09.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.862s CPU time. 2024-09-14T03:48:09.596 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 systemd[1]: Starting Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:09.966 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:09 smithi012 bash[83857]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:48:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:09 smithi001 ceph-mon[29360]: pgmap v29: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:48:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:09 smithi001 ceph-mon[40972]: pgmap v29: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:48:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:09 smithi012 ceph-mon[33640]: pgmap v29: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:48:10.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:10 smithi012 bash[83857]: Getting image source signatures 2024-09-14T03:48:10.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:10 smithi012 bash[83857]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:48:10.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:10 smithi012 bash[83857]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:48:10.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:10 smithi012 bash[83857]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:48:11.848 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:11 smithi012 bash[83857]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:48:11.849 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:11 smithi012 bash[83857]: Writing manifest to image destination 2024-09-14T03:48:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:11 smithi001 ceph-mon[29360]: pgmap v30: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:11 smithi001 ceph-mon[40972]: pgmap v30: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:12.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:11] "GET /metrics HTTP/1.1" 200 36193 "" "Prometheus/2.43.0" 2024-09-14T03:48:12.226 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:11 smithi012 ceph-mon[33640]: pgmap v30: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:12.226 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 podman[83857]: 2024-09-14 03:48:12.025324353 +0000 UTC m=+2.465404107 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:48:12.226 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 podman[83857]: 2024-09-14 03:48:12.098565859 +0000 UTC m=+2.538645654 container create a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:12.597 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 podman[83857]: 2024-09-14 03:48:12.325385281 +0000 UTC m=+2.765465035 container init a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:12.597 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 podman[83857]: 2024-09-14 03:48:12.331166316 +0000 UTC m=+2.771246070 container start a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:48:12.597 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.334Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.334Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.335Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.335Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.335Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:48:12.598 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:48:12.599 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:48:12.600 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:48:12.601 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b[83932]: ts=2024-09-14T03:48:12.336Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:48:12.602 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 bash[83857]: a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 2024-09-14T03:48:12.603 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 03:48:12 smithi012 systemd[1]: Started Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:13.387 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-14T03:48:13.388 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-14T03:48:13.681 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[29360]: pgmap v31: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[40972]: pgmap v31: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:13 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:13.823 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:13 smithi012 ceph-mon[33640]: pgmap v31: 89 pgs: 89 active+clean; 1.7 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:48:13.824 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:13 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.824 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:13 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:13.824 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:13 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:15.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:15 smithi012 ceph-mon[33640]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-14T03:48:15.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:15 smithi001 ceph-mon[29360]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-14T03:48:15.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:15 smithi001 ceph-mon[40972]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-14T03:48:16.495 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_0' removed 2024-09-14T03:48:16.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:16 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-14T03:48:16.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:16 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-14T03:48:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:16 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-14T03:48:17.281 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T03:48:17.283 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T03:48:17.283 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-14T03:48:17.283 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-14T03:48:17.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 363 op/s 2024-09-14T03:48:17.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:48:17.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 363 op/s 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.494 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:17 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:17.682 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 363 op/s 2024-09-14T03:48:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3772626642' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:48:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:17.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:17 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:18 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:18 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:18 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:18 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:18 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:18 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.754 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:19 smithi012 ceph-mon[33640]: pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:19.755 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:19 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.755 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:19 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.812 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[29360]: pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:19.813 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.813 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.813 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[40972]: pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:19.813 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:19.813 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:19 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.203 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_1' created 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-14T03:48:21.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-14T03:48:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:21 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:21.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:21] "GET /metrics HTTP/1.1" 200 33832 "" "Prometheus/2.43.0" 2024-09-14T03:48:22.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:22 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T03:48:22.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:22 smithi001 ceph-mon[29360]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:22.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:22 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T03:48:22.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:22 smithi001 ceph-mon[40972]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:22.506 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-14T03:48:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:22 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3499124289' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T03:48:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:22 smithi012 ceph-mon[33640]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:23.093 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[29360]: pgmap v38: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[29360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[29360]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[40972]: pgmap v38: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[40972]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:23 smithi001 ceph-mon[40972]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:23 smithi012 ceph-mon[33640]: pgmap v38: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:23 smithi012 ceph-mon[33640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:23 smithi012 ceph-mon[33640]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:24 smithi001 ceph-mon[29360]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:24 smithi001 ceph-mon[40972]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:24 smithi012 ceph-mon[33640]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:26.092 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-14T03:48:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: pgmap v41: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2098190316' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:26.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:26 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: pgmap v41: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2098190316' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: pgmap v41: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:26.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2098190316' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:26.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:26 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:27.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:27 smithi001 ceph-mon[40972]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:27.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:27 smithi001 ceph-mon[40972]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:27.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:27 smithi001 ceph-mon[29360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:27.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:27 smithi001 ceph-mon[29360]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:27.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:27 smithi012 ceph-mon[33640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:27.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:27 smithi012 ceph-mon[33640]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:27.970 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-14T03:48:27.974 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T03:48:27.974 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T03:48:28.032 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T03:48:28.032 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T03:48:28.032 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_101913 2024-09-14T03:48:28.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:48:28.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T03:48:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:28 smithi012 ceph-mon[33640]: pgmap v43: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:28 smithi012 ceph-mon[33640]: Upgrade: Updating prometheus.a 2024-09-14T03:48:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:28 smithi012 ceph-mon[33640]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:48:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:28 smithi012 ceph-mon[33640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:28 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[29360]: pgmap v43: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[29360]: Upgrade: Updating prometheus.a 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[29360]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[29360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[29360]: Cluster is now healthy 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[40972]: pgmap v43: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[40972]: Upgrade: Updating prometheus.a 2024-09-14T03:48:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[40972]: Deploying daemon prometheus.a on smithi012 2024-09-14T03:48:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[40972]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:28 smithi001 ceph-mon[40972]: Cluster is now healthy 2024-09-14T03:48:29.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3415 3399 212.426 212.438 0.00591513 0.00469649 2024-09-14T03:48:30.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 7129 7113 222.265 232.125 0.00577866 0.00448774 2024-09-14T03:48:30.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:30 smithi001 ceph-mon[29360]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:30 smithi001 ceph-mon[40972]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:30.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:30 smithi012 ceph-mon[33640]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:31.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 9956 9940 207.069 176.688 0.00607566 0.00482083 2024-09-14T03:48:32.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 13040 13024 203.484 192.75 0.00382403 0.00490746 2024-09-14T03:48:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:32 smithi001 ceph-mon[29360]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:32.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:32 smithi001 ceph-mon[40972]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:32.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:31] "GET /metrics HTTP/1.1" 200 33832 "" "Prometheus/2.43.0" 2024-09-14T03:48:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:32 smithi012 ceph-mon[33640]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:33.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 16060 16044 200.534 188.75 0.00655915 0.00497963 2024-09-14T03:48:33.513 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:48:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:33 smithi012 ceph-mon[33640]: pgmap v46: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[29360]: pgmap v46: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[40972]: pgmap v46: 89 pgs: 89 active+clean; 581 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:34.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 18770 18754 195.338 169.375 0.00559366 0.00511288 2024-09-14T03:48:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:34 smithi012 ceph-mon[33640]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:34 smithi012 ceph-mon[33640]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:34 smithi001 ceph-mon[29360]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:34 smithi001 ceph-mon[29360]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:34 smithi001 ceph-mon[40972]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.674 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:34 smithi001 ceph-mon[40972]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (6m) 15s ago 7m 27.4M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (6m) 17s ago 6m 90.1M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (5m) 15s ago 5m 19.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (5m) 17s ago 5m 14.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (51s) 17s ago 10m 454M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (90s) 15s ago 12m 510M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:48:34.675 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (12m) 15s ago 12m 56.1M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:48:34.676 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (11m) 17s ago 11m 44.4M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:48:34.676 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (11m) 15s ago 11m 45.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:48:34.676 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (29s) 15s ago 6m 8363k - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:48:34.676 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (22s) 17s ago 6m 3166k - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:48:34.676 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (10m) 15s ago 10m 394M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (10m) 15s ago 10m 259M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (9m) 15s ago 9m 334M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (9m) 15s ago 9m 180M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (9m) 17s ago 9m 395M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (8m) 17s ago 8m 431M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (8m) 17s ago 8m 225M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (7m) 17s ago 7m 272M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:48:34.677 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (58s) 17s ago 7m 44.6M - 2.43.0 a07b618ecd1d 53ce995a2d51 2024-09-14T03:48:35.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 21330 21314 190.288 160 0.00572116 0.00524901 2024-09-14T03:48:35.383 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:35.384 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:48:35.385 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:48:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:35 smithi012 ceph-mon[33640]: pgmap v47: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-14T03:48:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:35 smithi012 ceph-mon[33640]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:35 smithi001 ceph-mon[29360]: pgmap v47: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-14T03:48:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:35 smithi001 ceph-mon[29360]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:35 smithi001 ceph-mon[40972]: pgmap v47: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-14T03:48:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:35 smithi001 ceph-mon[40972]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr" 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading prometheus daemons", 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:48:35.982 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:48:36.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 15 23824 23809 185.992 155.938 0.00733541 0.00537108 2024-09-14T03:48:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:36 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/2583615060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:36 smithi012 ceph-mon[33640]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:36 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/2583615060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:36 smithi001 ceph-mon[29360]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:36.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:36 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/2583615060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:36.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:36 smithi001 ceph-mon[40972]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.033 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 26504 26488 183.929 167.438 0.0086608 0.00543106 2024-09-14T03:48:37.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:37 smithi012 ceph-mon[33640]: pgmap v48: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:48:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:37 smithi001 ceph-mon[29360]: pgmap v48: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:48:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:37 smithi001 ceph-mon[40972]: pgmap v48: 89 pgs: 89 active+clean; 916 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:48:38.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 28976 28960 180.984 154.5 0.00461297 0.00551961 2024-09-14T03:48:39.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 31288 31272 177.667 144.5 0.00694746 0.00562306 2024-09-14T03:48:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:39 smithi012 ceph-mon[33640]: pgmap v49: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:39.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:39 smithi001 ceph-mon[29360]: pgmap v49: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:39.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:39 smithi001 ceph-mon[40972]: pgmap v49: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:40.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 33675 33659 175.292 149.188 0.00906629 0.00569907 2024-09-14T03:48:41.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 35765 35749 171.854 130.625 0.00227169 0.00579006 2024-09-14T03:48:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:41 smithi012 ceph-mon[33640]: pgmap v50: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:41 smithi001 ceph-mon[29360]: pgmap v50: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:41 smithi001 ceph-mon[40972]: pgmap v50: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:42.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 37891 37875 169.069 132.875 0.00378 0.00590981 2024-09-14T03:48:42.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:41 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:41] "GET /metrics HTTP/1.1" 200 36125 "" "Prometheus/2.43.0" 2024-09-14T03:48:43.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 39385 39369 164.022 93.375 0.0804221 0.0060742 2024-09-14T03:48:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:43 smithi012 ceph-mon[33640]: pgmap v51: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:43 smithi001 ceph-mon[29360]: pgmap v51: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:43.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:43 smithi001 ceph-mon[40972]: pgmap v51: 89 pgs: 89 active+clean; 1.7 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-09-14T03:48:44.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 41082 41066 160.399 106.062 0.00558413 0.00622932 2024-09-14T03:48:45.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 43050 43034 158.199 123 0.00907002 0.00631564 2024-09-14T03:48:45.346 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 systemd[1]: Stopping Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:45 smithi001 ceph-mon[29360]: pgmap v52: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 MiB/s wr, 3.22k op/s 2024-09-14T03:48:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:45 smithi001 ceph-mon[40972]: pgmap v52: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 MiB/s wr, 3.22k op/s 2024-09-14T03:48:45.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-mon[33640]: pgmap v52: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 MiB/s wr, 3.22k op/s 2024-09-14T03:48:45.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.477Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[79891]: ts=2024-09-14T03:48:45.478Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:48:45.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:45 smithi012 podman[87985]: 2024-09-14 03:48:45.491080445 +0000 UTC m=+0.150909649 container died 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:46.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 44881 44865 155.767 114.438 0.00743275 0.00641422 2024-09-14T03:48:46.585 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 podman[87985]: 2024-09-14 03:48:46.265781889 +0000 UTC m=+0.925611100 container cleanup 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:46.585 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 bash[87985]: ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:48:46.841 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 podman[88002]: 2024-09-14 03:48:46.584201486 +0000 UTC m=+1.090097733 container remove 53ce995a2d515cc32bc9a6c189e84ea07e5b8002e79b72b8127a1e88bb9bed6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:46.841 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:48:46.841 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 systemd[1]: Stopped Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:46.841 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:46 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.670s CPU time. 2024-09-14T03:48:47.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 46848 46832 154.039 122.938 0.00679771 0.00648599 2024-09-14T03:48:47.422 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-mon[33640]: pgmap v53: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:48:47.423 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:47.423 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 systemd[1]: Starting Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:48:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:47 smithi001 ceph-mon[29360]: pgmap v53: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:48:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:47 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:47 smithi001 ceph-mon[40972]: pgmap v53: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:48:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:47 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:47.733 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 podman[88125]: 2024-09-14 03:48:47.328222011 +0000 UTC m=+0.029473534 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:48:47.733 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 podman[88125]: 2024-09-14 03:48:47.460988579 +0000 UTC m=+0.162240082 container create ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:48.034 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:48:48.034465+0000 min lat: 0.0010361 max lat: 0.116779 avg lat: 0.00656172 2024-09-14T03:48:48.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:48:48.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 48748 48732 152.274 118.75 0.00670559 0.00656172 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 podman[88125]: 2024-09-14 03:48:47.732465348 +0000 UTC m=+0.433716847 container init ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 podman[88125]: 2024-09-14 03:48:47.742857174 +0000 UTC m=+0.444108675 container start ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.781Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.781Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.781Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi012 (none))" 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.781Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.781Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.785Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:48:48.101 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.786Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.788Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.788Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.790Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.790Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.36µs 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.790Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.795Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.807Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.809Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.817Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.818Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.818Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=60.151µs wal_replay_duration=27.33228ms wbl_replay_duration=141ns total_replay_duration=27.511667ms 2024-09-14T03:48:48.102 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.824Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.824Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.824Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 bash[88125]: ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 systemd[1]: Started Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.848Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.883533ms db_storage=1.083µs remote_storage=1.368µs web_handler=388ns query_engine=826ns scrape=560.922µs scrape_sd=166.306µs notify=17.29µs notify_sd=15.831µs rules=22.596418ms tracing=7.89µs 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.849Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:48:48.103 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:48:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:48:47.849Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:48:49.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 50616 50600 150.581 116.75 0.00665085 0.00663496 2024-09-14T03:48:49.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:48 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:48 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:48 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:49.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:48 smithi012 ceph-mon[33640]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[29360]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:49.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:49.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:48 smithi001 ceph-mon[40972]: mgrmap e34: y(active, since 92s), standbys: x 2024-09-14T03:48:50.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 52470 52454 149.003 115.875 0.00658272 0.0067063 2024-09-14T03:48:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:49 smithi012 ceph-mon[33640]: pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2024-09-14T03:48:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:49 smithi001 ceph-mon[40972]: pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2024-09-14T03:48:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:49 smithi001 ceph-mon[29360]: pgmap v54: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2024-09-14T03:48:51.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 54276 54260 147.432 112.875 0.00740741 0.00677753 2024-09-14T03:48:52.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 56052 56036 145.914 111 0.00937479 0.00684859 2024-09-14T03:48:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:51 smithi001 ceph-mon[29360]: pgmap v55: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:51 smithi001 ceph-mon[40972]: pgmap v55: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:52.236 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:51 smithi012 ceph-mon[33640]: pgmap v55: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:53.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 57810 57794 144.472 109.875 0.00832132 0.00691759 2024-09-14T03:48:54.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 59487 59471 142.946 104.812 0.00968406 0.00699044 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[40972]: pgmap v56: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[29360]: pgmap v56: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:53 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:53 smithi012 ceph-mon[33640]: pgmap v56: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-14T03:48:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:53 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:53 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 61099 61083 141.383 100.75 0.00790403 0.00706742 2024-09-14T03:48:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:55 smithi012 ceph-mon[33640]: pgmap v57: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-14T03:48:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:55 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:55 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[40972]: pgmap v57: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[29360]: pgmap v57: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:55 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:48:56.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 62746 62730 140.01 102.938 0.0121412 0.00713646 2024-09-14T03:48:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:48:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:48:56] "GET /metrics HTTP/1.1" 200 36187 "" "Prometheus/2.51.0" 2024-09-14T03:48:57.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 64357 64341 138.653 100.688 0.0103304 0.00720743 2024-09-14T03:48:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:57 smithi012 ceph-mon[33640]: pgmap v58: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-14T03:48:57.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:57 smithi001 ceph-mon[40972]: pgmap v58: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-14T03:48:57.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:57 smithi001 ceph-mon[29360]: pgmap v58: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-14T03:48:58.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 65923 65907 137.294 97.875 0.0105405 0.0072779 2024-09-14T03:48:59.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 67399 67383 135.84 92.25 0.00802782 0.00735685 2024-09-14T03:48:59.786 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:48:59 smithi012 ceph-mon[33640]: pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T03:48:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:48:59 smithi001 ceph-mon[29360]: pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T03:48:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:48:59 smithi001 ceph-mon[40972]: pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T03:49:00.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 68873 68857 134.474 92.125 0.00598221 0.00743172 2024-09-14T03:49:01.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 70328 70312 133.154 90.9375 0.0187727 0.00750448 2024-09-14T03:49:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:01 smithi012 ceph-mon[33640]: pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:01 smithi001 ceph-mon[29360]: pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:01 smithi001 ceph-mon[40972]: pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:02.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 71672 71656 131.709 84 0.0104596 0.00758748 2024-09-14T03:49:03.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 73013 72997 130.34 83.8125 0.00444219 0.00766804 2024-09-14T03:49:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: pgmap v61: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: pgmap v61: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:03 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: pgmap v61: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:03.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:03 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:04.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 74280 74264 128.919 79.1875 0.0097724 0.00775203 2024-09-14T03:49:05.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 75599 75583 127.662 82.4375 0.00799708 0.00782785 2024-09-14T03:49:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:05 smithi012 ceph-mon[33640]: pgmap v62: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T03:49:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:05 smithi001 ceph-mon[29360]: pgmap v62: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T03:49:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:05 smithi001 ceph-mon[40972]: pgmap v62: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T03:49:06.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 76887 76871 126.421 80.5 0.0104086 0.00790468 2024-09-14T03:49:06.571 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:06 smithi001 ceph-mon[40972]: Upgrade: Updating alertmanager.a 2024-09-14T03:49:06.571 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:06 smithi001 ceph-mon[40972]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:49:06.571 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:06 smithi001 ceph-mon[29360]: Upgrade: Updating alertmanager.a 2024-09-14T03:49:06.571 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:06 smithi001 ceph-mon[29360]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:49:06.571 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:49:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:06 smithi012 ceph-mon[33640]: Upgrade: Updating alertmanager.a 2024-09-14T03:49:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:06 smithi012 ceph-mon[33640]: Deploying daemon alertmanager.a on smithi001 2024-09-14T03:49:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:06] "GET /metrics HTTP/1.1" 200 36189 "" "Prometheus/2.51.0" 2024-09-14T03:49:07.036 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 77961 77945 124.9 67.125 0.0197139 0.008001 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (6m) 47s ago 7m 27.4M - 0.25.0 c8568f914cd2 8aa6177b7ddd 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (7m) 13s ago 7m 90.1M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (6m) 47s ago 6m 19.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (6m) 13s ago 6m 14.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (84s) 13s ago 11m 454M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (2m) 47s ago 13m 510M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (13m) 47s ago 13m 56.1M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:49:07.517 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (11m) 13s ago 11m 45.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (11m) 47s ago 11m 45.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (62s) 47s ago 7m 8363k - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (55s) 13s ago 7m 11.6M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (11m) 47s ago 11m 394M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (10m) 47s ago 10m 259M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (10m) 47s ago 10m 334M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (9m) 47s ago 9m 180M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (9m) 13s ago 9m 342M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (9m) 13s ago 9m 366M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (8m) 13s ago 8m 213M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (8m) 13s ago 8m 265M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:49:07.518 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (20s) 13s ago 7m 34.9M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:49:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:07 smithi012 ceph-mon[33640]: pgmap v63: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:49:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:07 smithi012 ceph-mon[33640]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:07 smithi012 ceph-mon[33640]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[40972]: pgmap v63: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[40972]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[40972]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[29360]: pgmap v63: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[29360]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:07 smithi001 ceph-mon[29360]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:08.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:49:08.036377+0000 min lat: 0.0010361 max lat: 0.142972 avg lat: 0.00810523 2024-09-14T03:49:08.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:49:08.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 78931 78915 123.293 60.625 0.0132696 0.00810523 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:49:08.197 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:49:08.198 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:49:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:08 smithi001 ceph-mon[40972]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:08 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/38624284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:08 smithi001 ceph-mon[29360]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:08 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/38624284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:08 smithi012 ceph-mon[33640]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:08 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/38624284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:08.742 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:49:08.742 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:49:08.742 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:49:08.742 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr" 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading alertmanager daemons", 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:49:08.743 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:49:09.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 80098 80082 122.065 72.9375 0.00554293 0.00818734 2024-09-14T03:49:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:09 smithi012 ceph-mon[33640]: pgmap v64: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T03:49:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:09 smithi012 ceph-mon[33640]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:09 smithi001 ceph-mon[29360]: pgmap v64: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T03:49:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:09 smithi001 ceph-mon[29360]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:09 smithi001 ceph-mon[40972]: pgmap v64: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T03:49:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:09 smithi001 ceph-mon[40972]: from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:10.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 15 81412 81397 121.115 82.1875 0.00578007 0.0082514 2024-09-14T03:49:11.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 82542 82526 119.939 70.5625 0.0133234 0.00833272 2024-09-14T03:49:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:11 smithi012 ceph-mon[33640]: pgmap v65: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:11 smithi001 ceph-mon[29360]: pgmap v65: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:11 smithi001 ceph-mon[40972]: pgmap v65: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:12.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 83808 83792 119.011 79.125 0.00858564 0.00839721 2024-09-14T03:49:13.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 84988 84972 118.005 73.75 0.00913649 0.00846934 2024-09-14T03:49:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:13 smithi012 ceph-mon[33640]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:13 smithi001 ceph-mon[29360]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:13 smithi001 ceph-mon[40972]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:49:14.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 86290 86274 117.209 81.375 0.0128851 0.00852689 2024-09-14T03:49:15.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 87450 87434 116.257 72.5 0.013151 0.00859643 2024-09-14T03:49:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:15 smithi012 ceph-mon[33640]: pgmap v67: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-14T03:49:15.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:15 smithi001 ceph-mon[29360]: pgmap v67: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-14T03:49:15.640 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:15 smithi001 ceph-mon[40972]: pgmap v67: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-14T03:49:16.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 88530 88514 115.241 67.5 0.0103023 0.00867224 2024-09-14T03:49:16.911 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:16] "GET /metrics HTTP/1.1" 200 36189 "" "Prometheus/2.51.0" 2024-09-14T03:49:17.037 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 89675 89659 114.35 71.5625 0.00488477 0.00873921 2024-09-14T03:49:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:17 smithi001 ceph-mon[40972]: pgmap v68: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T03:49:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:17 smithi001 ceph-mon[29360]: pgmap v68: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T03:49:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:17 smithi012 ceph-mon[33640]: pgmap v68: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T03:49:18.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 90788 90772 113.454 69.5625 0.0219794 0.00880759 2024-09-14T03:49:18.198 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:17 smithi001 systemd[1]: Stopping Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:18 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:18 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:18.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[77826]: ts=2024-09-14T03:49:18.194Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:49:18.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:18 smithi001 podman[102676]: 2024-09-14 03:49:18.24816 +0000 UTC m=+0.214632649 container died 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:18 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:18.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:18 smithi001 podman[102676]: 2024-09-14 03:49:18.811277619 +0000 UTC m=+0.777750256 container cleanup 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:18.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:18 smithi001 bash[102676]: ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:49:19.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 91987 91971 112.699 74.9375 0.0395812 0.00886834 2024-09-14T03:49:19.319 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:19 smithi001 podman[102692]: 2024-09-14 03:49:19.035789252 +0000 UTC m=+0.782907392 container remove 8aa6177b7ddd7d9a32c438240c5638153fb33cefb46a299c70c228d55be66e25 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:19 smithi012 ceph-mon[33640]: pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:49:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:19 smithi001 ceph-mon[29360]: pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:49:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:19 smithi001 ceph-mon[40972]: pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T03:49:19.722 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:19 smithi001 podman[102692]: 2024-09-14 03:49:19.445361419 +0000 UTC m=+1.192479562 volume remove 88ab7845ea8682dd5160b388b7a68f1fa7e7713c66efc0f68196723cc713c09a 2024-09-14T03:49:20.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 93101 93085 111.87 69.625 0.00907099 0.00893195 2024-09-14T03:49:20.221 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:19 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:49:20.221 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:19 smithi001 systemd[1]: Stopped Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:20.221 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:19 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.858s CPU time. 2024-09-14T03:49:20.649 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:20 smithi001 systemd[1]: Starting Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:20.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:20 smithi001 podman[102806]: 2024-09-14 03:49:20.552281364 +0000 UTC m=+0.033588283 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-14T03:49:20.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:20 smithi001 podman[102806]: 2024-09-14 03:49:20.671665548 +0000 UTC m=+0.152972463 volume create 9aea3f7410907508d54b6c643ee1845eba8b24a669828f12b16c68f8931dbaf1 2024-09-14T03:49:20.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:20 smithi001 podman[102806]: 2024-09-14 03:49:20.789118795 +0000 UTC m=+0.270425713 container create 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:21.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 94200 94184 111.055 68.6875 0.0146292 0.00899962 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 podman[102806]: 2024-09-14 03:49:21.160155521 +0000 UTC m=+0.641462445 container init 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 podman[102806]: 2024-09-14 03:49:21.170995397 +0000 UTC m=+0.652302311 container start 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.192Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.193Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.195Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.1 port=9094 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.200Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.244Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:49:21.270 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.244Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:49:21.271 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.249Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:49:21.271 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:21.249Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:49:21.271 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 bash[102806]: 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e 2024-09-14T03:49:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:21 smithi012 ceph-mon[33640]: pgmap v70: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-mon[40972]: pgmap v70: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:21 smithi001 ceph-mon[29360]: pgmap v70: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:21.722 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:21 smithi001 systemd[1]: Started Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:22.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 15 95271 95256 110.239 67 0.00717729 0.00906627 2024-09-14T03:49:22.468 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.469 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.469 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:22.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:23.038 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 96372 96356 109.484 68.75 0.0137414 0.00912897 2024-09-14T03:49:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:23 smithi001 ceph-mon[40972]: pgmap v71: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:23 smithi001 ceph-mon[29360]: pgmap v71: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:23.472 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:23 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:23.200Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000539197s 2024-09-14T03:49:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:23 smithi012 ceph-mon[33640]: pgmap v71: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-14T03:49:24.039 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 97468 97452 108.751 68.5 0.0143854 0.00919056 2024-09-14T03:49:25.039 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 98561 98545 108.041 68.3125 0.0111291 0.00925002 2024-09-14T03:49:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:25 smithi001 ceph-mon[40972]: pgmap v72: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:49:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:25 smithi001 ceph-mon[29360]: pgmap v72: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:49:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:25 smithi012 ceph-mon[33640]: pgmap v72: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T03:49:26.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 99628 99612 107.328 66.6875 0.0191635 0.00931179 2024-09-14T03:49:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:26] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.51.0" 2024-09-14T03:49:27.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 100618 100602 106.557 61.875 0.0165 0.00937932 2024-09-14T03:49:28.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:49:28.039788+0000 min lat: 0.0010361 max lat: 0.142972 avg lat: 0.00944697 2024-09-14T03:49:28.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:49:28.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 101592 101576 105.796 60.875 0.0153637 0.00944697 2024-09-14T03:49:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:27 smithi012 ceph-mon[33640]: pgmap v73: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-14T03:49:28.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:27 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:28.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:27 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[29360]: pgmap v73: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-14T03:49:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:28.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[40972]: pgmap v73: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-14T03:49:28.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:28.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:27 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 102641 102625 105.136 65.5625 0.0171477 0.00950626 2024-09-14T03:49:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.198 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.198 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.198 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.199 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.199 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.199 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.199 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:29.199 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:30.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 103757 103741 104.565 69.75 0.0109498 0.00955823 2024-09-14T03:49:30.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:29 smithi012 ceph-mon[33640]: pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:49:30.404 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:29 smithi001 ceph-mon[40972]: pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:49:30.405 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:29 smithi001 ceph-mon[29360]: pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:49:31.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 104818 104802 103.958 66.3125 0.0211175 0.00961411 2024-09-14T03:49:31.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:49:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:49:31.205Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004815051s 2024-09-14T03:49:32.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 105836 105820 103.328 63.625 0.0221161 0.00967325 2024-09-14T03:49:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:31 smithi012 ceph-mon[33640]: pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:31 smithi001 ceph-mon[40972]: pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:31 smithi001 ceph-mon[29360]: pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:33.040 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 106838 106822 102.701 62.625 0.0101723 0.00973126 2024-09-14T03:49:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:32 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:33.423 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:32 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:33.423 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:32 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:34.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 107877 107861 102.129 64.9375 0.0228044 0.0097864 2024-09-14T03:49:34.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:34.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.249 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.249 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.249 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:33 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:34.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:34.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:33 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:35.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 108949 108933 101.605 67 0.00775036 0.00983659 2024-09-14T03:49:35.218 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:34 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:34 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:34 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:49:36.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 110038 110022 101.111 68.0625 0.00989105 0.00988523 2024-09-14T03:49:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:36 smithi012 ceph-mon[33640]: pgmap v77: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:49:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:36 smithi001 ceph-mon[40972]: pgmap v77: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:49:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:36 smithi001 ceph-mon[29360]: pgmap v77: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:49:37.014 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:36] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-14T03:49:37.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 110955 110939 100.477 57.3125 0.00733399 0.00994745 2024-09-14T03:49:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:37 smithi012 ceph-mon[33640]: Upgrade: Updating grafana.a 2024-09-14T03:49:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:37 smithi012 ceph-mon[33640]: Deploying daemon grafana.a on smithi012 2024-09-14T03:49:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:37 smithi001 ceph-mon[40972]: Upgrade: Updating grafana.a 2024-09-14T03:49:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:37 smithi001 ceph-mon[40972]: Deploying daemon grafana.a on smithi012 2024-09-14T03:49:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:37 smithi001 ceph-mon[29360]: Upgrade: Updating grafana.a 2024-09-14T03:49:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:37 smithi001 ceph-mon[29360]: Deploying daemon grafana.a on smithi012 2024-09-14T03:49:38.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 112003 111987 99.9768 65.5 0.0227768 0.00999707 2024-09-14T03:49:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:38 smithi012 ceph-mon[33640]: pgmap v78: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-14T03:49:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:38 smithi001 ceph-mon[29360]: pgmap v78: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-14T03:49:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:38 smithi001 ceph-mon[40972]: pgmap v78: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-14T03:49:39.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 113004 112988 99.4498 62.5625 0.0231504 0.0100497 2024-09-14T03:49:39.322 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:49:40.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 113905 113889 98.8506 56.3125 0.0288584 0.0101112 2024-09-14T03:49:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:40 smithi012 ceph-mon[33640]: pgmap v79: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (19s) 13s ago 8m 17.4M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (7m) 12s ago 7m 89.7M - 9.4.7 954c08fa6188 8da64d8412e7 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (6m) 13s ago 6m 20.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (6m) 12s ago 6m 14.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (117s) 12s ago 12m 455M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:49:40.460 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (2m) 13s ago 13m 516M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (13m) 13s ago 13m 59.4M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (12m) 12s ago 12m 47.3M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (12m) 13s ago 12m 47.3M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (95s) 13s ago 7m 8128k - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (88s) 12s ago 7m 10.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (11m) 13s ago 11m 368M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (11m) 13s ago 11m 284M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (10m) 13s ago 10m 396M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (10m) 13s ago 10m 380M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (10m) 12s ago 10m 381M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (9m) 12s ago 9m 414M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (9m) 12s ago 9m 261M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:49:40.461 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (8m) 12s ago 8m 326M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:49:40.462 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (53s) 12s ago 8m 43.0M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:49:40.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:40 smithi001 ceph-mon[40972]: pgmap v79: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T03:49:40.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:40 smithi001 ceph-mon[29360]: pgmap v79: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T03:49:41.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 114815 114799 98.2755 56.875 0.0198919 0.0101695 2024-09-14T03:49:41.127 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:49:41.127 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:49:41.127 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:49:41.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:49:41.128 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:49:41.129 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:49:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:41 smithi012 ceph-mon[33640]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:41 smithi012 ceph-mon[33640]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:41 smithi001 ceph-mon[29360]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:41 smithi001 ceph-mon[29360]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:41 smithi001 ceph-mon[40972]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:41 smithi001 ceph-mon[40972]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.710 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:49:41.710 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:49:41.710 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr" 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading grafana daemons", 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:49:41.711 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:49:42.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 115718 115702 97.7101 56.4375 0.0230532 0.0102294 2024-09-14T03:49:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:42 smithi012 ceph-mon[33640]: pgmap v80: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:42 smithi012 ceph-mon[33640]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:42 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/310872332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[40972]: pgmap v80: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[40972]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/310872332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[29360]: pgmap v80: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[29360]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:42 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/310872332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:43.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 116636 116620 97.1722 57.375 0.00952139 0.0102858 2024-09-14T03:49:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:43 smithi012 ceph-mon[33640]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:43.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:43 smithi001 ceph-mon[40972]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:43.373 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:43 smithi001 ceph-mon[29360]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:44.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 117602 117586 96.688 60.375 0.0109017 0.0103375 2024-09-14T03:49:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:44 smithi012 ceph-mon[33640]: pgmap v81: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:44.388 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:44 smithi001 ceph-mon[29360]: pgmap v81: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:44.388 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:44 smithi001 ceph-mon[40972]: pgmap v81: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T03:49:45.041 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 118510 118494 96.1693 56.75 0.0144399 0.0103924 2024-09-14T03:49:46.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 119388 119372 95.6398 54.875 0.00888208 0.0104506 2024-09-14T03:49:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:46 smithi012 ceph-mon[33640]: pgmap v82: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:49:46.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:46 smithi001 ceph-mon[29360]: pgmap v82: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:49:46.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:46 smithi001 ceph-mon[40972]: pgmap v82: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:49:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:46] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-14T03:49:47.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 120345 120329 95.1863 59.8125 0.00811181 0.0105006 2024-09-14T03:49:48.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:49:48.041637+0000 min lat: 0.0010361 max lat: 0.142972 avg lat: 0.0105467 2024-09-14T03:49:48.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:49:48.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 121335 121319 94.7698 61.875 0.0143156 0.0105467 2024-09-14T03:49:48.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:48 smithi012 ceph-mon[33640]: pgmap v83: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:49:48.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:48 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:48 smithi001 ceph-mon[40972]: pgmap v83: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:49:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:48 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:48 smithi001 ceph-mon[29360]: pgmap v83: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:49:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:48 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:49.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 122291 122275 94.3374 59.75 0.0126514 0.0105951 2024-09-14T03:49:50.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 123273 123257 93.9354 61.375 0.0079897 0.0106399 2024-09-14T03:49:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:50 smithi012 ceph-mon[33640]: pgmap v84: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T03:49:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:50 smithi001 ceph-mon[40972]: pgmap v84: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T03:49:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:50 smithi001 ceph-mon[29360]: pgmap v84: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T03:49:51.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 124209 124193 93.5084 58.5 0.0198953 0.0106891 2024-09-14T03:49:52.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 125177 125161 93.1154 60.5 0.0258584 0.0107341 2024-09-14T03:49:52.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:52 smithi012 systemd[1]: Stopping Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:52 smithi012 ceph-mon[33640]: pgmap v85: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:52 smithi001 ceph-mon[29360]: pgmap v85: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:52 smithi001 ceph-mon[40972]: pgmap v85: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:52.846 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:52 smithi012 podman[92620]: 2024-09-14 03:49:52.532923293 +0000 UTC m=+0.155602111 container died 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, name=ubi8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc.) 2024-09-14T03:49:53.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 126147 126131 92.7331 60.625 0.0236404 0.0107783 2024-09-14T03:49:53.168 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:52 smithi012 podman[92620]: 2024-09-14 03:49:52.95761429 +0000 UTC m=+0.580293106 container cleanup 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , name=ubi8, distribution-scope=public, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-type=git, release=239.1651231664) 2024-09-14T03:49:53.169 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:52 smithi012 bash[92620]: ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:49:53.508 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 podman[92636]: 2024-09-14 03:49:53.167658439 +0000 UTC m=+0.633894781 container remove 8da64d8412e7f6d2e31eb657abf428c33bede00690c7a318ae16e61818d6cb32 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-09-14T03:49:53.508 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:49:53.823 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:49:53.823 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 systemd[1]: Stopped Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:53.823 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service: Consumed 10.050s CPU time. 2024-09-14T03:49:54.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 127038 127022 92.3023 55.6875 0.00736728 0.0108286 2024-09-14T03:49:54.080 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:53 smithi012 systemd[1]: Starting Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:54.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 podman[92761]: 2024-09-14 03:49:54.012660044 +0000 UTC m=+0.026521388 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-14T03:49:54.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 podman[92761]: 2024-09-14 03:49:54.144720777 +0000 UTC m=+0.158582117 container create d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:49:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-mon[33640]: pgmap v86: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:54.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:54 smithi001 ceph-mon[40972]: pgmap v86: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:54 smithi001 ceph-mon[29360]: pgmap v86: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2024-09-14T03:49:54.707 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 podman[92761]: 2024-09-14 03:49:54.442311587 +0000 UTC m=+0.456172930 container init d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:49:54.707 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 podman[92761]: 2024-09-14 03:49:54.449640904 +0000 UTC m=+0.463502244 container start d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:49:54.707 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 bash[92761]: d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 2024-09-14T03:49:54.707 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 systemd[1]: Started Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680011592Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-14T03:49:54Z 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680493816Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680538826Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680565462Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680589406Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680616891Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680650151Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680686017Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680718817Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:49:54.708 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680749711Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680783726Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680814367Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.68084276Z level=info msg=Target target=[all] 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680881368Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680914065Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680945738Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.680975423Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.681006849Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=settings t=2024-09-14T03:49:54.681035976Z level=info msg="App mode production" 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore t=2024-09-14T03:49:54.681523289Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore t=2024-09-14T03:49:54.681589181Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.682367394Z level=info msg="Starting DB migrations" 2024-09-14T03:49:54.709 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.705632991Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.717319795Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=11.683052ms 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.719811208Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.720262801Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=455.378µs 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.721252198Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.723078152Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=1.82514ms 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.724330247Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.735020671Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=10.686533ms 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.736652129Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-14T03:49:54.962 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.747241801Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=10.586823ms 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.748328896Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.759512763Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=11.179407ms 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.760486543Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.76080634Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=320.443µs 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.761503344Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.762994259Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=1.493515ms 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.763890132Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.765388231Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=1.49681ms 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.766234439Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.778035627Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=11.796915ms 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.778998341Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-14T03:49:54.963 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.789416582Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=10.413667ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.790354235Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.790500005Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=145.625µs 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.791401159Z level=info msg="Executing migration" id="permission kind migration" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.802399848Z level=info msg="Migration successfully executed" id="permission kind migration" duration=10.99305ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.80503786Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.81439362Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=9.356339ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.815150196Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.829561484Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=14.40547ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.830642163Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.832175675Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=1.533558ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.832923176Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.834316202Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=1.392442ms 2024-09-14T03:49:54.964 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.83532159Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.836604993Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=1.28361ms 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.837216188Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.837246137Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=30.39µs 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.837872122Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.839237425Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=1.363678ms 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.840030276Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.841495789Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=1.457248ms 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.842208131Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.870084813Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=27.872807ms 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.871174252Z level=info msg="Executing migration" id="create correlation v2" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.872702685Z level=info msg="Migration successfully executed" id="create correlation v2" duration=1.527773ms 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.873606717Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-14T03:49:54.965 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.874959259Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=1.353031ms 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.880039669Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.881610381Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=1.573182ms 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.883102504Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.884472412Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=1.371612ms 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.885693043Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.886008738Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=316.467µs 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.887496572Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.888935591Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=1.438249ms 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.890120309Z level=info msg="Executing migration" id="add provisioning column" 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.899669886Z level=info msg="Migration successfully executed" id="add provisioning column" duration=9.545985ms 2024-09-14T03:49:54.966 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.901812582Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.902249419Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=463.614µs 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.903080013Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.903225279Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=145.105µs 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.904328129Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.914988956Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=10.656095ms 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.916040358Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.916519699Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=479.866µs 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.91746852Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.917867112Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=398.776µs 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.918682104Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.920597639Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=1.914036ms 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.921371364Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-14T03:49:54.967 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.922696637Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=1.324578ms 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.923317587Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.924589785Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=1.271439ms 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.928544697Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.929296242Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=752.394µs 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.929881935Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.930275591Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=396.476µs 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.930852231Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.932023369Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=1.171306ms 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.932543117Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.933696216Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=1.152396ms 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.934266312Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.935429644Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=1.162616ms 2024-09-14T03:49:54.968 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.9360382Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.941647957Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=5.608648ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.943996105Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.945561588Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=1.565875ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.946240654Z level=info msg="Executing migration" id="create anon_device table" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.947361246Z level=info msg="Migration successfully executed" id="create anon_device table" duration=1.120622ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.94808984Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.949336959Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=1.247111ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.952422957Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.954033717Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=1.611493ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.955321657Z level=info msg="Executing migration" id="create signing_key table" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.956531943Z level=info msg="Migration successfully executed" id="create signing_key table" duration=1.209882ms 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.957644521Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-14T03:49:54.969 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.959175308Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=1.531852ms 2024-09-14T03:49:55.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 127970 127954 91.9108 58.25 0.00932219 0.0108745 2024-09-14T03:49:55.251 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.965229889Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-14T03:49:55.251 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.966556898Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=1.328429ms 2024-09-14T03:49:55.251 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.968381857Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.969360049Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=983.874µs 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.970168436Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.981088218Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=10.916081ms 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.982042386Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.985451382Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=3.409953ms 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.988066949Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.990345257Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=2.278158ms 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.991741203Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.993400351Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=1.655577ms 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.996043597Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.997458573Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=1.41652ms 2024-09-14T03:49:55.252 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.998081873Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:54.999518581Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=1.436438ms 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.000153296Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.001692195Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=1.537196ms 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.002353694Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.003886598Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=1.531893ms 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.004720174Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.006045053Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=1.325981ms 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.006693689Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.007581881Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=888.268µs 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.008492085Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.008645679Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=154.353µs 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.009366332Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.020406416Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=11.035221ms 2024-09-14T03:49:55.253 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.021200453Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.031413129Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=10.210259ms 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.032837352Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.033422862Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=587.211µs 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=migrator t=2024-09-14T03:49:55.034215653Z level=info msg="migrations completed" performed=65 skipped=482 duration=328.712704ms 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore t=2024-09-14T03:49:55.035720977Z level=info msg="Created default organization" 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=secrets t=2024-09-14T03:49:55.039821907Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugin.store t=2024-09-14T03:49:55.069068399Z level=info msg="Loading plugins..." 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=local.finder t=2024-09-14T03:49:55.144944006Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugin.store t=2024-09-14T03:49:55.145195368Z level=info msg="Plugins loaded" count=55 duration=76.128012ms 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=query_data t=2024-09-14T03:49:55.148265328Z level=info msg="Query Service initialization" 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=live.push_http t=2024-09-14T03:49:55.153813079Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.migration t=2024-09-14T03:49:55.155859216Z level=info msg=Starting 2024-09-14T03:49:55.254 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.state.manager t=2024-09-14T03:49:55.164947891Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=infra.usagestats.collector t=2024-09-14T03:49:55.167133488Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.datasources t=2024-09-14T03:49:55.181427913Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.datasources t=2024-09-14T03:49:55.181999483Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.alerting t=2024-09-14T03:49:55.246834234Z level=info msg="starting to provision alerting" 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.alerting t=2024-09-14T03:49:55.246913203Z level=info msg="finished to provision alerting" 2024-09-14T03:49:55.255 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.state.manager t=2024-09-14T03:49:55.248955426Z level=info msg="Warming state cache for startup" 2024-09-14T03:49:55.583 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafanaStorageLogger t=2024-09-14T03:49:55.249931441Z level=info msg="Storage starting" 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=http.server t=2024-09-14T03:49:55.250463567Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=http.server t=2024-09-14T03:49:55.250801831Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.dashboard t=2024-09-14T03:49:55.251683351Z level=info msg="starting to provision dashboards" 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:49:55.280707399Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugins.update.checker t=2024-09-14T03:49:55.358701444Z level=info msg="Update check succeeded" duration=108.356672ms 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana.update.checker t=2024-09-14T03:49:55.365067427Z level=info msg="Update check succeeded" duration=113.920708ms 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.state.manager t=2024-09-14T03:49:55.380297391Z level=info msg="State cache has been initialized" states=0 duration=131.338176ms 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ngalert.scheduler t=2024-09-14T03:49:55.380336621Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ticker t=2024-09-14T03:49:55.381213116Z level=info msg=starting first_tick=2024-09-14T03:50:00Z 2024-09-14T03:49:55.584 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore.transactions t=2024-09-14T03:49:55.384744393Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:49:55.585 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore.transactions t=2024-09-14T03:49:55.439985044Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:49:55.846 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=provisioning.dashboard t=2024-09-14T03:49:55.65664839Z level=info msg="finished to provision dashboards" 2024-09-14T03:49:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-mon[33640]: pgmap v87: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T03:49:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[29360]: pgmap v87: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T03:49:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[40972]: pgmap v87: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T03:49:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:49:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:55 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:56.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 128843 128827 91.4863 54.5625 0.0167061 0.0109256 2024-09-14T03:49:56.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana-apiserver t=2024-09-14T03:49:55.927011041Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:49:56.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:49:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana-apiserver t=2024-09-14T03:49:55.927758909Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:49:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:49:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:49:56] "GET /metrics HTTP/1.1" 200 36206 "" "Prometheus/2.51.0" 2024-09-14T03:49:57.043 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 129782 129766 91.1178 58.6875 0.00805746 0.0109697 2024-09-14T03:49:57.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:57 smithi001 ceph-mon[40972]: pgmap v88: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-09-14T03:49:57.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:57 smithi001 ceph-mon[29360]: pgmap v88: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-09-14T03:49:58.019 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:57 smithi012 ceph-mon[33640]: pgmap v88: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-09-14T03:49:58.042 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 90 3 130671 130668 90.7317 56.375 0.0146129 0.0110169 2024-09-14T03:49:59.043 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0108 2024-09-14T03:49:59.043 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 130671 2024-09-14T03:49:59.043 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T03:49:59.043 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 90.7328 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 39.9485 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 232.125 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 54.5625 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1451 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 639.176 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3714 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 873 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0110169 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.00771429 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 0.142972 2024-09-14T03:49:59.044 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.0010361 2024-09-14T03:49:59.049 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T03:49:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:49:59 smithi001 ceph-mon[29360]: pgmap v89: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T03:49:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:49:59 smithi001 ceph-mon[40972]: pgmap v89: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T03:50:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:49:59 smithi012 ceph-mon[33640]: pgmap v89: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T03:50:00.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:00 smithi001 ceph-mon[29360]: overall HEALTH_OK 2024-09-14T03:50:00.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:00 smithi001 ceph-mon[40972]: overall HEALTH_OK 2024-09-14T03:50:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:00 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[29360]: pgmap v90: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[40972]: pgmap v90: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:02.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:01 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:01 smithi012 ceph-mon[33640]: pgmap v90: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:01 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:01 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:01 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:02.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:01 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:02 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:02 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:02 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:03 smithi001 ceph-mon[29360]: pgmap v91: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:03 smithi001 ceph-mon[40972]: pgmap v91: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:04.245 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:03 smithi012 ceph-mon[33640]: pgmap v91: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2024-09-14T03:50:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:05 smithi001 ceph-mon[29360]: pgmap v92: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.10k op/s 2024-09-14T03:50:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:05 smithi001 ceph-mon[40972]: pgmap v92: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.10k op/s 2024-09-14T03:50:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:05 smithi012 ceph-mon[33640]: pgmap v92: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.10k op/s 2024-09-14T03:50:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:50:06] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.51.0" 2024-09-14T03:50:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:50:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:50:07.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:07 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:50:07.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:50:07.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:50:07.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:50:07.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:50:07.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:07.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:07.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:07.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:07 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:08 smithi012 ceph-mon[33640]: pgmap v93: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 702 op/s 2024-09-14T03:50:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:08 smithi012 ceph-mon[33640]: Upgrade: Finalizing container_image settings 2024-09-14T03:50:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:08 smithi012 ceph-mon[33640]: Upgrade: Complete! 2024-09-14T03:50:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:08 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[29360]: pgmap v93: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 702 op/s 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[29360]: Upgrade: Finalizing container_image settings 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[29360]: Upgrade: Complete! 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[40972]: pgmap v93: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 702 op/s 2024-09-14T03:50:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[40972]: Upgrade: Finalizing container_image settings 2024-09-14T03:50:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[40972]: Upgrade: Complete! 2024-09-14T03:50:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:08 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:10 smithi012 ceph-mon[33640]: pgmap v94: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 1.10k op/s 2024-09-14T03:50:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:10 smithi001 ceph-mon[29360]: pgmap v94: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 1.10k op/s 2024-09-14T03:50:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:10 smithi001 ceph-mon[40972]: pgmap v94: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 1.10k op/s 2024-09-14T03:50:12.318 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:12 smithi001 ceph-mon[29360]: pgmap v95: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:12.319 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:12 smithi001 ceph-mon[40972]: pgmap v95: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:12 smithi012 ceph-mon[33640]: pgmap v95: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:13.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-14T03:50:13.563 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:50:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:14 smithi012 ceph-mon[33640]: pgmap v96: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:14 smithi012 ceph-mon[33640]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:14 smithi001 ceph-mon[40972]: pgmap v96: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:14 smithi001 ceph-mon[40972]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:14 smithi001 ceph-mon[29360]: pgmap v96: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 721 op/s 2024-09-14T03:50:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:14 smithi001 ceph-mon[29360]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:15.929 INFO:teuthology.orchestra.run.smithi001.stdout:wait for minority of mons to upgrade 2024-09-14T03:50:16.335 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:16 smithi001 ceph-mon[40972]: pgmap v97: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.12k op/s 2024-09-14T03:50:16.335 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:16 smithi001 ceph-mon[29360]: pgmap v97: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.12k op/s 2024-09-14T03:50:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:16 smithi012 ceph-mon[33640]: pgmap v97: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.12k op/s 2024-09-14T03:50:16.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-14T03:50:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:50:16] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.51.0" 2024-09-14T03:50:17.162 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:50:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:18 smithi012 ceph-mon[33640]: pgmap v98: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-14T03:50:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:18 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:18.456 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:18 smithi001 ceph-mon[40972]: pgmap v98: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-14T03:50:18.456 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:18 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:18.456 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:18 smithi001 ceph-mon[29360]: pgmap v98: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-09-14T03:50:18.456 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:18 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:20 smithi012 ceph-mon[33640]: pgmap v99: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-14T03:50:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:20 smithi001 ceph-mon[40972]: pgmap v99: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-14T03:50:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:20 smithi001 ceph-mon[29360]: pgmap v99: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-14T03:50:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:21 smithi001 ceph-mon[40972]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:21 smithi001 ceph-mon[29360]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:21 smithi012 ceph-mon[33640]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:22.036 INFO:teuthology.orchestra.run.smithi001.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: pgmap v100: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:22.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:22.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:22.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:22 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: pgmap v100: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: pgmap v100: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:22.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:22.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:22.398 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:22 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:22.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:50:23.199 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:50:23.460 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:23 smithi001 ceph-mon[29360]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:23.460 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:23 smithi001 ceph-mon[40972]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:23 smithi012 ceph-mon[33640]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:24 smithi001 ceph-mon[29360]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:24 smithi001 ceph-mon[29360]: pgmap v101: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:24 smithi001 ceph-mon[40972]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:24 smithi001 ceph-mon[40972]: pgmap v101: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:24 smithi012 ceph-mon[33640]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:50:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:24 smithi012 ceph-mon[33640]: pgmap v101: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-14T03:50:26.185 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: pgmap v102: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:50:26.185 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: pgmap v102: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:26.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.187 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:50:26.187 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:50:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: pgmap v102: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:50:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:50:26.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:26 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:50:26.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:50:26] "GET /metrics HTTP/1.1" 200 36202 "" "Prometheus/2.51.0" 2024-09-14T03:50:27.268 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[40972]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[40972]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[40972]: Upgrade: Setting container_image for all mgr 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[40972]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[29360]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:50:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[29360]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:50:27.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[29360]: Upgrade: Setting container_image for all mgr 2024-09-14T03:50:27.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:27 smithi001 ceph-mon[29360]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:50:27.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:27 smithi012 ceph-mon[33640]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:50:27.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:27 smithi012 ceph-mon[33640]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:50:27.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:27 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all mgr 2024-09-14T03:50:27.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:27 smithi012 ceph-mon[33640]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:50:28.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[29360]: pgmap v103: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T03:50:28.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:28.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:50:28.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:50:28.425 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[29360]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:28.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[40972]: pgmap v103: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T03:50:28.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:28.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:50:28.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:50:28.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:28 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:28.427 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:50:28.427 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (67s) 61s ago 8m 17.4M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:50:28.427 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (34s) 27s ago 8m 42.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:50:28.427 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (7m) 61s ago 7m 20.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:50:28.427 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (7m) 27s ago 7m 14.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (2m) 27s ago 12m 455M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (3m) 61s ago 14m 516M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (14m) 61s ago 14m 59.4M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 12f8e6dc9182 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (13m) 27s ago 13m 48.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (13m) 61s ago 13m 47.3M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (2m) 61s ago 8m 8128k - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (2m) 27s ago 8m 10.9M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (12m) 61s ago 12m 368M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (12m) 61s ago 12m 284M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (11m) 61s ago 11m 396M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (11m) 61s ago 11m 380M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (10m) 27s ago 10m 423M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:50:28.428 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (10m) 27s ago 10m 454M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:50:28.429 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (10m) 27s ago 10m 303M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:50:28.429 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (9m) 27s ago 9m 384M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:50:28.429 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (101s) 27s ago 9m 43.0M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:50:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:28 smithi012 ceph-mon[33640]: pgmap v103: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T03:50:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' 2024-09-14T03:50:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:50:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:50:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:28 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:29.094 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:50:29.095 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:50:29.096 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:50:29.096 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:50:29.096 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:50:29.350 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[40972]: from='client.24922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[40972]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[40972]: Upgrade: Updating mon.a 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[40972]: Deploying daemon mon.a on smithi001 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/1834129503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[29360]: from='client.24922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[29360]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[29360]: Upgrade: Updating mon.a 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[29360]: Deploying daemon mon.a on smithi001 2024-09-14T03:50:29.351 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:29 smithi001 ceph-mon[29360]: from='client.? 172.21.15.1:0/1834129503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:29 smithi012 ceph-mon[33640]: from='client.24922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:29 smithi012 ceph-mon[33640]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:29 smithi012 ceph-mon[33640]: Upgrade: Updating mon.a 2024-09-14T03:50:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:29 smithi012 ceph-mon[33640]: Deploying daemon mon.a on smithi001 2024-09-14T03:50:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:29 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1834129503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:29.664 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:50:29.664 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "0/3 daemons upgraded", 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:50:29.665 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:50:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-mon[29360]: pgmap v104: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:50:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-mon[29360]: from='client.25049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 systemd[1]: Stopping Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:30.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-mon[40972]: pgmap v104: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:50:30.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-mon[40972]: from='client.25049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:30 smithi012 ceph-mon[33640]: pgmap v104: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:50:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:30 smithi012 ceph-mon[33640]: from='client.25049 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:30.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[29337]: 2024-09-14T03:50:30.667+0000 7f859428a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:50:30.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[29337]: 2024-09-14T03:50:30.667+0000 7f859428a640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:30 smithi001 podman[107222]: 2024-09-14 03:50:30.802864681 +0000 UTC m=+0.283009047 container died 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE Bus STOPPING 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE Bus STOPPED 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE Bus STARTING 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE Serving on http://:::9283 2024-09-14T03:50:30.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:30] ENGINE Bus STARTED 2024-09-14T03:50:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:31 smithi001 podman[107222]: 2024-09-14 03:50:31.98228822 +0000 UTC m=+1.462432579 container cleanup 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, RELEASE=reef-e4e227b) 2024-09-14T03:50:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:31 smithi001 bash[107222]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:50:32.597 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:32 smithi001 podman[107249]: 2024-09-14 03:50:32.326237271 +0000 UTC m=+1.511581789 container remove 12f8e6dc91822362d1f30536ed3cc78d2fb24bf81d2366d0e93f010ff65178b0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:50:32.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:32 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:50:32.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:32 smithi001 systemd[1]: Stopped Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:32.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:32 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service: Consumed 18.342s CPU time. 2024-09-14T03:50:33.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:32 smithi001 systemd[1]: Starting Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:33.583 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 podman[107364]: 2024-09-14 03:50:33.162267317 +0000 UTC m=+0.035517978 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:50:33.583 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 podman[107364]: 2024-09-14 03:50:33.285938776 +0000 UTC m=+0.159189422 container create 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:50:33.583 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 podman[107364]: 2024-09-14 03:50:33.582290793 +0000 UTC m=+0.455541434 container init 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 podman[107364]: 2024-09-14 03:50:33.595386789 +0000 UTC m=+0.468637435 container start 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 6 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: pidfile_write: ignore empty --pid-file 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: load: jerasure load: lrc 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Git sha 0 2024-09-14T03:50:33.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: DB SUMMARY 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: DB Session ID: OXBHXZOJDB24RL0YB4J5 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1518 Bytes 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 3522899 ; 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:50:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.env: 0x5561f9484120 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.info_log: 0x5561fa791700 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.statistics: (nil) 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.use_fsync: 0 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:50:33.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.db_log_dir: 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.wal_dir: 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:50:33.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.write_buffer_manager: 0x5561fa794000 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:50:33.976 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.unordered_write: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.row_cache: None 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.wal_filter: None 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:50:33.977 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:50:33.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.wal_compression: 0 2024-09-14T03:50:33.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:50:33.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:50:33.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:50:33.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:50:33.979 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_open_files: -1 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:50:33.980 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Compression algorithms supported: 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kZSTD supported: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:50:33.981 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.merge_operator: 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_filter: None 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5561fa791320) 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:50:33.982 INFO:journalctl@ceph.mon.a.smithi001.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_type: 0 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: data_block_index_type: 0 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_shortening: 1 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: checksum: 4 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: no_block_cache: 0 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache: 0x5561fa785610 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_options: 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: capacity : 536870912 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: num_shard_bits : 4 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: strict_capacity_limit : 0 2024-09-14T03:50:33.983 INFO:journalctl@ceph.mon.a.smithi001.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_cache_compressed: (nil) 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: persistent_cache: (nil) 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_size: 4096 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_size_deviation: 10 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_restart_interval: 16 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: index_block_restart_interval: 1 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: metadata_block_size: 4096 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: partition_filters: 0 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: use_delta_encoding: 1 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: filter_policy: bloomfilter 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: whole_key_filtering: 1 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: verify_compression: 0 2024-09-14T03:50:33.984 INFO:journalctl@ceph.mon.a.smithi001.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: format_version: 5 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: enable_index_compression: 1 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: block_align: 0 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: max_auto_readahead_size: 262144 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: prepopulate_block_cache: 0 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression: NoCompression 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:50:33.985 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.num_levels: 7 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:50:33.986 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:50:33.987 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:50:33.988 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:50:33.989 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:50:33.989 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:50:33.989 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:50:33.989 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:50:33.989 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:50:33.990 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:50:33.991 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.ttl: 2592000 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:50:33.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:50:33.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12131, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 384c41e5-9a68-4872-8924-4edf54067175 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285833651470, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285833670536, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 3175849, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12132, "largest_seqno": 14973, "table_properties": {"data_size": 3163905, "index_size": 7441, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3461, "raw_key_size": 30702, "raw_average_key_size": 22, "raw_value_size": 3136525, "raw_average_value_size": 2272, "num_data_blocks": 337, "num_entries": 1380, "num_filter_entries": 1380, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285833, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "384c41e5-9a68-4872-8924-4edf54067175", "db_session_id": "OXBHXZOJDB24RL0YB4J5", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285833670757, "job": 1, "event": "recovery_finished"} 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-09-14T03:50:33.994 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 bash[107364]: 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5561fa7b4e00 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: DB pointer 0x5561fa85c000 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** DB Stats ** 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:50:33.995 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: L0 1/0 3.03 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 334.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: L6 1/0 7.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: Sum 2/0 10.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 334.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 334.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:50:33.996 INFO:journalctl@ceph.mon.a.smithi001.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 334.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: Cumulative compaction: 0.00 GB write, 80.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:50:33.997 INFO:journalctl@ceph.mon.a.smithi001.stdout: Interval compaction: 0.00 GB write, 80.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: Block cache BinnedLRUCache@0x5561fa785610#6 capacity: 512.00 MB usage: 64.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: Block cache entry stats(count,size,portion): DataBlock(4,27.31 KB,0.00520945%) FilterBlock(2,13.06 KB,0.00249147%) IndexBlock(2,24.16 KB,0.00460744%) Misc(1,0.00 KB,0%) 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 systemd[1]: Started Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: starting mon.a rank 0 at public addrs [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] at bind addrs [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???) e3 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).mds e9 new map 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).mds e9 print_map 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: e9 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:50:33.998 INFO:journalctl@ceph.mon.a.smithi001.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: legacy client fscid: 1 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: Filesystem 'foo' (1) 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: fs_name foo 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: epoch 8 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: created 2024-09-14T03:42:34.635441+0000 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: modified 2024-09-14T03:42:47.984273+0000 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: tableserver 0 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: root 0 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: session_timeout 60 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: session_autoclose 300 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: max_file_size 1099511627776 2024-09-14T03:50:33.999 INFO:journalctl@ceph.mon.a.smithi001.stdout: max_xattr_size 65536 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: required_client_features {} 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: last_failure 0 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: last_failure_osd_epoch 0 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: max_mds 1 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: in 0 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: up {0=24557} 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: failed 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: damaged 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: stopped 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: data_pools [4] 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: metadata_pool 3 2024-09-14T03:50:34.000 INFO:journalctl@ceph.mon.a.smithi001.stdout: inline_data disabled 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: balancer 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: bal_rank_mask -1 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: standby_count_wanted 1 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: [mds.foo.smithi001.ytjgbj{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: Standby daemons: 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout: [mds.foo.smithi012.nnhswt{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:50:34.001 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:50:34.002 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:50:34.002 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:33 smithi001 ceph-mon[107399]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-14T03:50:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:50:36] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-09-14T03:50:40.346 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: ignoring --setuser ceph since I am not root 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: ignoring --setgroup ceph since I am not root 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mgr[80363]: -- 172.21.15.12:0/3912728297 <== mon.1 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e7d89cb0e0 con 0x55e7d89a9400 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: pgmap v105: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: pgmap v106: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: pgmap v107: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: pgmap v108: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: pgmap v109: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: mon.a calling monitor election 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: monmap epoch 3 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:50:40.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: min_mon_release 18 (reef) 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: election_strategy: 1 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: mgrmap e34: y(active, since 3m), standbys: x 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-14T03:50:40.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:40 smithi012 ceph-mon[33640]: from='mgr.24860 ' entity='' 2024-09-14T03:50:40.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ignoring --setuser ceph since I am not root 2024-09-14T03:50:40.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ignoring --setgroup ceph since I am not root 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: pgmap v105: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: pgmap v106: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: pgmap v107: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: pgmap v108: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:50:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: pgmap v109: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: mon.a calling monitor election 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: monmap epoch 3 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: min_mon_release 18 (reef) 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: election_strategy: 1 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:50:40.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: mgrmap e34: y(active, since 3m), standbys: x 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: overall HEALTH_OK 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[40972]: from='mgr.24860 ' entity='' 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: pgmap v105: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:40.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: pgmap v106: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 172.21.15.1:0/3888324373' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: pgmap v107: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: pgmap v108: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: pgmap v109: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: mon.a calling monitor election 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: monmap epoch 3 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: min_mon_release 18 (reef) 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: election_strategy: 1 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:50:40.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: mgrmap e34: y(active, since 3m), standbys: x 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: overall HEALTH_OK 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-14T03:50:40.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:40 smithi001 ceph-mon[107399]: from='mgr.24860 ' entity='' 2024-09-14T03:50:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: Standby manager daemon x restarted 2024-09-14T03:50:48.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: Standby manager daemon x started 2024-09-14T03:50:48.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:48.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:48.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-mon[33640]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:50:47] ENGINE Bus STARTING 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: CherryPy Checker: 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: The Application mounted at '' has an empty config. 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:50:47] ENGINE Serving on http://:::9283 2024-09-14T03:50:48.098 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x[80340]: [14/Sep/2024:03:50:47] ENGINE Bus STARTED 2024-09-14T03:50:48.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.829Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.830Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.099 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.832Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.834Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.834Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.837Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.100 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 03:50:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T03:50:47.837Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: Standby manager daemon x restarted 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: Standby manager daemon x started 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[107399]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: Standby manager daemon x restarted 2024-09-14T03:50:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: Standby manager daemon x started 2024-09-14T03:50:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:48.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:47 smithi001 ceph-mon[40972]: from='mgr.? 172.21.15.12:0/916167003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:48 smithi012 ceph-mon[33640]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-14T03:50:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:48 smithi001 ceph-mon[107399]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-14T03:50:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:48 smithi001 ceph-mon[40972]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-14T03:50:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: Active manager daemon y restarted 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: Activating manager daemon y 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: mgrmap e37: y(active, starting, since 0.0135637s), standbys: x 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:49.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[107399]: Manager daemon y is now available 2024-09-14T03:50:49.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: Active manager daemon y restarted 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: Activating manager daemon y 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: mgrmap e37: y(active, starting, since 0.0135637s), standbys: x 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:49.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:49.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:49 smithi001 ceph-mon[40972]: Manager daemon y is now available 2024-09-14T03:50:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: Active manager daemon y restarted 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: Activating manager daemon y 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: mgrmap e37: y(active, starting, since 0.0135637s), standbys: x 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.nnhswt"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi001.ytjgbj"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:50.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:50.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:49 smithi012 ceph-mon[33640]: Manager daemon y is now available 2024-09-14T03:50:50.238 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:50:50.218Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephPoolGrowthWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.1:8443: connect: connection refused" 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:50] ENGINE Bus STARTING 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: CherryPy Checker: 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: The Application mounted at '' has an empty config. 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.038+0000 7fc9b3f23640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:50] ENGINE Serving on http://:::9283 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:50:50] ENGINE Bus STARTED 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.155+0000 7fc988b3f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.155+0000 7fc988b3f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.155+0000 7fc988b3f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.239 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.155+0000 7fc988b3f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.155+0000 7fc988b3f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.172+0000 7fc984b37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.172+0000 7fc984b37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.172+0000 7fc984b37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.172+0000 7fc984b37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:50.240 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:50.172+0000 7fc984b37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: mgrmap e38: y(active, since 1.03361s), standbys: x 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: mgrmap e38: y(active, since 1.03361s), standbys: x 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:51.028 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:50 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:50:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: mgrmap e38: y(active, since 1.03361s), standbys: x 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:50 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:50:51.971 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T03:50:51.683Z caller=notify.go:860 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephPoolGrowthWarning\"}" msg="Notify success" attempts=3 duration=271.105768ms 2024-09-14T03:50:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:51 smithi012 ceph-mon[33640]: pgmap v3: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:51 smithi012 ceph-mon[33640]: pgmap v4: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:51 smithi012 ceph-mon[33640]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[107399]: pgmap v3: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[107399]: pgmap v4: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[107399]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[40972]: pgmap v3: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[40972]: pgmap v4: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:51 smithi001 ceph-mon[40972]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:50:52.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:52.552+0000 7fc9637bb640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:50:52] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: Traceback (most recent call last): 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._connections.run(self.expiration_interval) 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._run(expiration_interval) 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:52.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: s = self.context.wrap_socket( 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: return self.sslsocket_class._create( 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self.do_handshake() 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: self._sslobj.do_handshake() 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:52.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:50:54.308 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:54.308 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: [14/Sep/2024:03:50:52] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:50:54.308 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: [14/Sep/2024:03:50:52] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.308 INFO:journalctl@ceph.mon.c.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.308 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.309 INFO:journalctl@ceph.mon.c.smithi001.stdout: self.do_handshake() 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: [14/Sep/2024:03:50:52] ENGINE Bus STARTED 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[40972]: pgmap v5: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:54.310 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: [14/Sep/2024:03:50:52] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: [14/Sep/2024:03:50:52] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.311 INFO:journalctl@ceph.mon.a.smithi001.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout: self.do_handshake() 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: [14/Sep/2024:03:50:52] ENGINE Bus STARTED 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:50:54.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:54 smithi001 ceph-mon[107399]: pgmap v5: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: [14/Sep/2024:03:50:52] ENGINE Serving on https://172.21.15.1:7150 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: [14/Sep/2024:03:50:52] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.347 INFO:journalctl@ceph.mon.b.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self.do_handshake() 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.1:8765 2024-09-14T03:50:54.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: [14/Sep/2024:03:50:52] ENGINE Bus STARTED 2024-09-14T03:50:54.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:50:54.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:54 smithi012 ceph-mon[33640]: pgmap v5: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-14T03:50:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:56 smithi012 ceph-mon[33640]: pgmap v6: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2024-09-14T03:50:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:56 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:56 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[40972]: pgmap v6: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[107399]: pgmap v6: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:50:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:50:56] "GET /metrics HTTP/1.1" 200 33742 "" "Prometheus/2.51.0" 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[107399]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 KiB/s rd, 444 B/s wr, 543 op/s 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.437 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:50:57 smithi001 ceph-mon[40972]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 KiB/s rd, 444 B/s wr, 543 op/s 2024-09-14T03:50:57.530 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.530 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.530 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.530 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.531 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.531 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:50:57.531 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:50:57 smithi012 ceph-mon[33640]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 KiB/s rd, 444 B/s wr, 543 op/s 2024-09-14T03:51:00.298 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:51:00.298 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:00 smithi001 ceph-mon[40972]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 KiB/s rd, 344 B/s wr, 421 op/s 2024-09-14T03:51:00.299 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:00 smithi001 ceph-mon[107399]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 KiB/s rd, 344 B/s wr, 421 op/s 2024-09-14T03:51:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:00 smithi012 ceph-mon[33640]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 KiB/s rd, 344 B/s wr, 421 op/s 2024-09-14T03:51:01.471 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:51:01.471 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (100s) 5s ago 9m 20.9M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:51:01.471 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (67s) 5s ago 9m 76.5M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:51:01.471 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (8m) 5s ago 8m 20.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (8m) 5s ago 8m 14.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (3m) 5s ago 13m 451M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (3m) 5s ago 15m 493M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (28s) 5s ago 15m 36.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (13m) 5s ago 13m 55.1M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (13m) 5s ago 13m 54.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 7b713104fec0 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (2m) 5s ago 9m 11.9M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (2m) 5s ago 8m 11.8M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (13m) 5s ago 13m 402M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (12m) 5s ago 12m 339M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:51:01.472 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (12m) 5s ago 12m 480M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (11m) 5s ago 11m 498M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (11m) 5s ago 11m 436M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (11m) 5s ago 11m 461M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (10m) 5s ago 10m 317M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (10m) 5s ago 10m 401M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:51:01.473 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (2m) 5s ago 9m 43.1M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:51:02.149 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:51:02.149 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:51:02.149 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 12, 2024-09-14T03:51:02.150 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:51:02.151 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:51:02.151 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[40972]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[40972]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[40972]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 16 KiB/s rd, 281 B/s wr, 718 op/s 2024-09-14T03:51:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[107399]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[107399]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:01 smithi001 ceph-mon[107399]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 16 KiB/s rd, 281 B/s wr, 718 op/s 2024-09-14T03:51:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:01 smithi012 ceph-mon[33640]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:01 smithi012 ceph-mon[33640]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:01 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:01 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:01 smithi012 ceph-mon[33640]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 16 KiB/s rd, 281 B/s wr, 718 op/s 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "1/3 daemons upgraded", 2024-09-14T03:51:02.741 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "", 2024-09-14T03:51:02.742 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:51:02.742 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:51:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:02 smithi001 ceph-mon[107399]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3197863758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:02 smithi001 ceph-mon[40972]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:03.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:02 smithi001 ceph-mon[40972]: from='client.? 172.21.15.1:0/3197863758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:02 smithi012 ceph-mon[33640]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:02 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3197863758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:04.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:04.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:03 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:03 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:04 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:51:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:51:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.conf 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-14T03:51:05.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:05.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:04 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[107399]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:51:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:05 smithi001 ceph-mon[40972]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: Updating smithi001:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: Updating smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: Updating smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 652 op/s 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:51:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:05 smithi012 ceph-mon[33640]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:51:06] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.51.0" 2024-09-14T03:51:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:08 smithi012 ceph-mon[33640]: pgmap v12: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 982 op/s 2024-09-14T03:51:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:08 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:08 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:08 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:08 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[107399]: pgmap v12: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 982 op/s 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[40972]: pgmap v12: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 982 op/s 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[40972]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:08 smithi001 ceph-mon[40972]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:09 smithi012 ceph-mon[33640]: Upgrade: Updating mon.c 2024-09-14T03:51:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:09 smithi012 ceph-mon[33640]: Deploying daemon mon.c on smithi001 2024-09-14T03:51:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-mon[40972]: Upgrade: Updating mon.c 2024-09-14T03:51:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-mon[40972]: Deploying daemon mon.c on smithi001 2024-09-14T03:51:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 systemd[1]: Stopping Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-mon[107399]: Upgrade: Updating mon.c 2024-09-14T03:51:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-mon[107399]: Deploying daemon mon.c on smithi001 2024-09-14T03:51:10.216 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:09] ENGINE Bus STOPPING 2024-09-14T03:51:10.216 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:10 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:51:10.217 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c[40949]: 2024-09-14T03:51:09.762+0000 7f940d78f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:51:10.217 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c[40949]: 2024-09-14T03:51:09.762+0000 7f940d78f640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:51:10.217 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:09 smithi001 podman[111996]: 2024-09-14 03:51:09.999171105 +0000 UTC m=+0.383607007 container died 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:51:10.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:10 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:10] ENGINE Bus STOPPED 2024-09-14T03:51:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 podman[111996]: 2024-09-14 03:51:10.40810607 +0000 UTC m=+0.792541972 container cleanup 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:51:10.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 bash[111996]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c 2024-09-14T03:51:10.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 podman[112010]: 2024-09-14 03:51:10.668536071 +0000 UTC m=+0.665959146 container remove 7b713104fec0ab1cd9ca30eb4af211c5933a74338ef45dcf018cc69e5efdbe7e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b) 2024-09-14T03:51:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-14T03:51:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 systemd[1]: Stopped Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:10 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.c.service: Consumed 9.898s CPU time. 2024-09-14T03:51:11.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:11 smithi001 systemd[1]: Starting Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:11 smithi001 podman[112131]: 2024-09-14 03:51:11.597117142 +0000 UTC m=+0.056889072 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:51:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:11 smithi001 podman[112131]: 2024-09-14 03:51:11.720023054 +0000 UTC m=+0.179794975 container create a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, RELEASE=main-a278231, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 podman[112131]: 2024-09-14 03:51:12.023767988 +0000 UTC m=+0.483539921 container init a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 podman[112131]: 2024-09-14 03:51:12.034296577 +0000 UTC m=+0.494068499 container start a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: pidfile_write: ignore empty --pid-file 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: load: jerasure load: lrc 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Git sha 0 2024-09-14T03:51:12.475 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: DB SUMMARY 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: DB Session ID: YZWQD1UUVRU470R98T9C 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 8532834 ; 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:51:12.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.env: 0x556e2522e120 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.info_log: 0x556e25ec1700 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.statistics: (nil) 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.use_fsync: 0 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:51:12.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.db_log_dir: 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.wal_dir: 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:51:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.write_buffer_manager: 0x556e25ec5b80 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:51:12.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.unordered_write: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.row_cache: None 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.wal_filter: None 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:51:12.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.wal_compression: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:51:12.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:51:12.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_open_files: -1 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Compression algorithms supported: 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kZSTD supported: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:51:12.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.merge_operator: 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_filter: None 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:51:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556e25ec1320) 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_type: 0 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: data_block_index_type: 0 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_shortening: 1 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: checksum: 4 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: no_block_cache: 0 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache: 0x556e25eb5610 2024-09-14T03:51:12.485 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_options: 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: capacity : 536870912 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: num_shard_bits : 4 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: strict_capacity_limit : 0 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_cache_compressed: (nil) 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: persistent_cache: (nil) 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_size: 4096 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_size_deviation: 10 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_restart_interval: 16 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: index_block_restart_interval: 1 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: metadata_block_size: 4096 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: partition_filters: 0 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: use_delta_encoding: 1 2024-09-14T03:51:12.486 INFO:journalctl@ceph.mon.c.smithi001.stdout: filter_policy: bloomfilter 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: whole_key_filtering: 1 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: verify_compression: 0 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: format_version: 5 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: enable_index_compression: 1 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: block_align: 0 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: max_auto_readahead_size: 262144 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: prepopulate_block_cache: 0 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:51:12.487 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression: NoCompression 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.num_levels: 7 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:12.488 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:51:12.489 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:51:12.490 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:51:12.491 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:51:12.491 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:51:12.491 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:51:12.492 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:51:12.493 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.ttl: 2592000 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:51:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12740, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 23c7004b-485e-4e17-9495-92dd0d4b99b5 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285872091187, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:51:12.495 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285872120415, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5238413, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12745, "largest_seqno": 16457, "table_properties": {"data_size": 5223901, "index_size": 9369, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4101, "raw_key_size": 36563, "raw_average_key_size": 22, "raw_value_size": 5191139, "raw_average_value_size": 3180, "num_data_blocks": 421, "num_entries": 1632, "num_filter_entries": 1632, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285872, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "23c7004b-485e-4e17-9495-92dd0d4b99b5", "db_session_id": "YZWQD1UUVRU470R98T9C", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285872120561, "job": 1, "event": "recovery_finished"} 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556e25ee4e00 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: DB pointer 0x556e25fe6000 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** DB Stats ** 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:12.496 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: L0 1/0 5.00 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 485.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: L6 1/0 7.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Sum 2/0 12.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 485.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 485.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:12.497 INFO:journalctl@ceph.mon.c.smithi001.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 485.3 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:51:12.498 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: Cumulative compaction: 0.00 GB write, 119.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: Interval compaction: 0.00 GB write, 119.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: Block cache BinnedLRUCache@0x556e25eb5610#7 capacity: 512.00 MB usage: 14.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,10.08 KB,0.00192225%) Misc(1,0.00 KB,0%) 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: starting mon.c rank 2 at public addrs [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] at bind addrs [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???) e3 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).mds e9 new map 2024-09-14T03:51:12.499 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).mds e9 print_map 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: e9 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: legacy client fscid: 1 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: Filesystem 'foo' (1) 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: fs_name foo 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: epoch 8 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: created 2024-09-14T03:42:34.635441+0000 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: modified 2024-09-14T03:42:47.984273+0000 2024-09-14T03:51:12.500 INFO:journalctl@ceph.mon.c.smithi001.stdout: tableserver 0 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: root 0 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: session_timeout 60 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: session_autoclose 300 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: max_file_size 1099511627776 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: max_xattr_size 65536 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: required_client_features {} 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: last_failure 0 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: last_failure_osd_epoch 0 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: max_mds 1 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: in 0 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: up {0=24557} 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: failed 2024-09-14T03:51:12.501 INFO:journalctl@ceph.mon.c.smithi001.stdout: damaged 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: stopped 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: data_pools [4] 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: metadata_pool 3 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: inline_data disabled 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: balancer 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: bal_rank_mask -1 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: standby_count_wanted 1 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: [mds.foo.smithi001.ytjgbj{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: Standby daemons: 2024-09-14T03:51:12.502 INFO:journalctl@ceph.mon.c.smithi001.stdout: 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout: [mds.foo.smithi012.nnhswt{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).osd e86 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 ceph-mon[112165]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 bash[112131]: a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 2024-09-14T03:51:12.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:12 smithi001 systemd[1]: Started Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:14.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:51:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=infra.usagestats t=2024-09-14T03:51:14.26032871Z level=info msg="Usage stats are ready to report" 2024-09-14T03:51:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-mon[107399]: pgmap v13: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-14T03:51:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:17.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:17.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:17.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:17] ENGINE Bus STARTING 2024-09-14T03:51:17.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:17] ENGINE Serving on http://:::9283 2024-09-14T03:51:17.723 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:51:17] ENGINE Bus STARTED 2024-09-14T03:51:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:17 smithi012 ceph-mon[33640]: pgmap v13: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-14T03:51:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:17 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:17 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:17 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: mon.c calling monitor election 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: mon.b calling monitor election 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: mon.a calling monitor election 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: monmap epoch 3 2024-09-14T03:51:18.624 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: min_mon_release 18 (reef) 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: election_strategy: 1 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: mgrmap e40: y(active, since 29s), standbys: x 2024-09-14T03:51:18.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[107399]: overall HEALTH_OK 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: pgmap v13: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: mon.c calling monitor election 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: mon.b calling monitor election 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: mon.a calling monitor election 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: monmap epoch 3 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:51:18.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: min_mon_release 18 (reef) 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: election_strategy: 1 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: mgrmap e40: y(active, since 29s), standbys: x 2024-09-14T03:51:18.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:18 smithi001 ceph-mon[112165]: overall HEALTH_OK 2024-09-14T03:51:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: mon.c calling monitor election 2024-09-14T03:51:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: mon.b calling monitor election 2024-09-14T03:51:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: mon.a calling monitor election 2024-09-14T03:51:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: monmap epoch 3 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: last_changed 2024-09-14T03:37:24.339528+0000 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: min_mon_release 18 (reef) 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: election_strategy: 1 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: mgrmap e40: y(active, since 29s), standbys: x 2024-09-14T03:51:18.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:18 smithi012 ceph-mon[33640]: overall HEALTH_OK 2024-09-14T03:51:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:19 smithi001 ceph-mon[107399]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-14T03:51:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:19 smithi001 ceph-mon[112165]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-14T03:51:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:19 smithi012 ceph-mon[33640]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-14T03:51:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:20 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:21 smithi001 ceph-mon[107399]: pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:21 smithi001 ceph-mon[112165]: pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:21 smithi012 ceph-mon[33640]: pgmap v19: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:22.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:51:22.142+0000 7fc9b3f23640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:51:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:23 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:23 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:23 smithi012 ceph-mon[33640]: pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:51:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[112165]: pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:51:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:23.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:23 smithi001 ceph-mon[107399]: pgmap v20: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:51:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:24 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:24 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:24 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:24 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:24 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:24 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:25 smithi012 ceph-mon[33640]: pgmap v21: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-09-14T03:51:25.661 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:25 smithi001 ceph-mon[107399]: pgmap v21: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-09-14T03:51:25.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:25 smithi001 ceph-mon[112165]: pgmap v21: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-09-14T03:51:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:51:26] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.51.0" 2024-09-14T03:51:28.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:28 smithi012 ceph-mon[33640]: pgmap v22: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:28 smithi001 ceph-mon[107399]: pgmap v22: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:28 smithi001 ceph-mon[112165]: pgmap v22: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: pgmap v23: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:51:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: pgmap v23: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:30.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:30 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: pgmap v23: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:30 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:51:31.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:51:31.476 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:51:31.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:51:31.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:51:31.477 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:51:31.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.481 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:51:31.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:51:31.483 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.485 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:51:31.486 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.487 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.488 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.489 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:31 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:51:31.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:51:31.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:51:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:31.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:31.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:31.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:31 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all grafana 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all loki 2024-09-14T03:51:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all promtail 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Finalizing container_image settings 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: Upgrade: Complete! 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[107399]: pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all grafana 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all loki 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all promtail 2024-09-14T03:51:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Finalizing container_image settings 2024-09-14T03:51:32.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: Upgrade: Complete! 2024-09-14T03:51:32.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:32 smithi001 ceph-mon[112165]: pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:51:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:51:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:51:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all grafana 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all loki 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all promtail 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Finalizing container_image settings 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: Upgrade: Complete! 2024-09-14T03:51:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:32 smithi012 ceph-mon[33640]: pgmap v24: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:51:34.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T03:51:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:34 smithi001 ceph-mon[107399]: pgmap v25: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-14T03:51:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:34 smithi001 ceph-mon[112165]: pgmap v25: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-14T03:51:34.510 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:34 smithi012 ceph-mon[33640]: pgmap v25: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-14T03:51:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[107399]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:35.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[112165]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:35.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:35 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:35 smithi012 ceph-mon[33640]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:35 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:35 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:51:36.272 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:36 smithi001 ceph-mon[107399]: pgmap v26: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-14T03:51:36.273 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:36 smithi001 ceph-mon[112165]: pgmap v26: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-14T03:51:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:36 smithi012 ceph-mon[33640]: pgmap v26: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-14T03:51:36.842 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:51:36] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-09-14T03:51:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:37 smithi001 ceph-mon[112165]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-09-14T03:51:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:37 smithi001 ceph-mon[107399]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-09-14T03:51:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:37 smithi012 ceph-mon[33640]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-09-14T03:51:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:40 smithi012 ceph-mon[33640]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-14T03:51:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:40 smithi001 ceph-mon[112165]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-14T03:51:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:40 smithi001 ceph-mon[107399]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-14T03:51:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:42 smithi012 ceph-mon[33640]: pgmap v29: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:42 smithi001 ceph-mon[112165]: pgmap v29: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:42 smithi001 ceph-mon[107399]: pgmap v29: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:51:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:44 smithi001 ceph-mon[112165]: pgmap v30: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:44 smithi001 ceph-mon[107399]: pgmap v30: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:44 smithi012 ceph-mon[33640]: pgmap v30: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:46 smithi001 ceph-mon[107399]: pgmap v31: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:46 smithi001 ceph-mon[112165]: pgmap v31: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:46 smithi012 ceph-mon[33640]: pgmap v31: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-14T03:51:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:51:46] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-09-14T03:51:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:48 smithi001 ceph-mon[107399]: pgmap v32: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:51:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:48 smithi001 ceph-mon[112165]: pgmap v32: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:51:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:48 smithi012 ceph-mon[33640]: pgmap v32: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:51:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:50 smithi001 ceph-mon[112165]: pgmap v33: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-14T03:51:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:50 smithi001 ceph-mon[107399]: pgmap v33: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-14T03:51:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:50 smithi012 ceph-mon[33640]: pgmap v33: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-14T03:51:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:52 smithi001 ceph-mon[112165]: pgmap v34: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:52 smithi001 ceph-mon[107399]: pgmap v34: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:52 smithi012 ceph-mon[33640]: pgmap v34: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:54 smithi001 ceph-mon[107399]: pgmap v35: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:54 smithi001 ceph-mon[112165]: pgmap v35: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:54 smithi012 ceph-mon[33640]: pgmap v35: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:56 smithi001 ceph-mon[112165]: pgmap v36: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:56 smithi001 ceph-mon[107399]: pgmap v36: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:56 smithi012 ceph-mon[33640]: pgmap v36: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-14T03:51:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:51:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:51:56] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.51.0" 2024-09-14T03:51:58.398 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:58 smithi012 ceph-mon[33640]: pgmap v37: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:58 smithi001 ceph-mon[112165]: pgmap v37: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:58 smithi001 ceph-mon[107399]: pgmap v37: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:51:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:51:59 smithi001 ceph-mon[112165]: pgmap v38: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-14T03:51:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:51:59 smithi001 ceph-mon[107399]: pgmap v38: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-14T03:52:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:51:59 smithi012 ceph-mon[33640]: pgmap v38: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-14T03:52:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:02 smithi001 ceph-mon[112165]: pgmap v39: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:52:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:02 smithi001 ceph-mon[107399]: pgmap v39: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:52:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:02 smithi012 ceph-mon[33640]: pgmap v39: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T03:52:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:04 smithi001 ceph-mon[112165]: pgmap v40: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:04 smithi001 ceph-mon[107399]: pgmap v40: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:04 smithi012 ceph-mon[33640]: pgmap v40: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:05.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:05 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:06.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:06 smithi001 ceph-mon[107399]: pgmap v41: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:06.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:06 smithi001 ceph-mon[112165]: pgmap v41: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:06 smithi012 ceph-mon[33640]: pgmap v41: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:06] "GET /metrics HTTP/1.1" 200 36189 "" "Prometheus/2.51.0" 2024-09-14T03:52:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:08 smithi001 ceph-mon[112165]: pgmap v42: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:08 smithi001 ceph-mon[107399]: pgmap v42: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:08 smithi012 ceph-mon[33640]: pgmap v42: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:10 smithi001 ceph-mon[112165]: pgmap v43: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:10 smithi001 ceph-mon[107399]: pgmap v43: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:10 smithi012 ceph-mon[33640]: pgmap v43: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:12 smithi001 ceph-mon[112165]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:12 smithi001 ceph-mon[107399]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:12 smithi012 ceph-mon[33640]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:14 smithi001 ceph-mon[107399]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:14 smithi001 ceph-mon[112165]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:14 smithi012 ceph-mon[33640]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:15 smithi001 ceph-mon[112165]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:15 smithi001 ceph-mon[107399]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:15 smithi012 ceph-mon[33640]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T03:52:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:16] "GET /metrics HTTP/1.1" 200 36189 "" "Prometheus/2.51.0" 2024-09-14T03:52:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:18 smithi001 ceph-mon[112165]: pgmap v47: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:18 smithi001 ceph-mon[107399]: pgmap v47: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:18 smithi012 ceph-mon[33640]: pgmap v47: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-14T03:52:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:20 smithi001 ceph-mon[112165]: pgmap v48: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:20 smithi001 ceph-mon[107399]: pgmap v48: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:20 smithi012 ceph-mon[33640]: pgmap v48: 89 pgs: 89 active+clean; 840 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-14T03:52:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:20 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:22.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:22 smithi001 ceph-mon[112165]: pgmap v49: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-14T03:52:22.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:22 smithi001 ceph-mon[112165]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T03:52:22.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:22 smithi001 ceph-mon[107399]: pgmap v49: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-14T03:52:22.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:22 smithi001 ceph-mon[107399]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T03:52:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:22 smithi012 ceph-mon[33640]: pgmap v49: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-14T03:52:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:22 smithi012 ceph-mon[33640]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T03:52:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:24 smithi001 ceph-mon[107399]: pgmap v50: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:24 smithi001 ceph-mon[112165]: pgmap v50: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:24 smithi012 ceph-mon[33640]: pgmap v50: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:26 smithi001 ceph-mon[112165]: pgmap v51: 89 pgs: 89 active+clean; 574 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:26 smithi001 ceph-mon[107399]: pgmap v51: 89 pgs: 89 active+clean; 574 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:26 smithi012 ceph-mon[33640]: pgmap v51: 89 pgs: 89 active+clean; 574 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-09-14T03:52:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:26] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.51.0" 2024-09-14T03:52:28.397 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:28 smithi012 ceph-mon[33640]: pgmap v52: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:28 smithi001 ceph-mon[112165]: pgmap v52: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:28 smithi001 ceph-mon[107399]: pgmap v52: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:29.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:29 smithi001 ceph-mon[112165]: pgmap v53: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 718 op/s 2024-09-14T03:52:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:29 smithi001 ceph-mon[107399]: pgmap v53: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 718 op/s 2024-09-14T03:52:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:29 smithi012 ceph-mon[33640]: pgmap v53: 89 pgs: 89 active+clean; 301 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 718 op/s 2024-09-14T03:52:30.665 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 130671 objects 2024-09-14T03:52:30.665 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :151.615 2024-09-14T03:52:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:30 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:31 smithi012 ceph-mon[33640]: pgmap v54: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:31 smithi001 ceph-mon[107399]: pgmap v54: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:31 smithi001 ceph-mon[112165]: pgmap v54: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 1.09k op/s 2024-09-14T03:52:32.029 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-14T03:52:32.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-14T03:52:32.573 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:34.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:34 smithi001 ceph-mon[107399]: pgmap v55: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:34.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:34 smithi001 ceph-mon[112165]: pgmap v55: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:34 smithi012 ceph-mon[33640]: pgmap v55: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:35.107 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_1' removed 2024-09-14T03:52:35.383 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:35.383 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:52:35.383 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:35.383 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:52:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:35 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:35 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:52:36.034 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T03:52:36.035 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T03:52:36.035 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-14T03:52:36.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[112165]: pgmap v56: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[112165]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[107399]: pgmap v56: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[107399]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:52:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:36 smithi012 ceph-mon[33640]: pgmap v56: 89 pgs: 89 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-14T03:52:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:36 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/3982704050' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:52:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:36 smithi012 ceph-mon[33640]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:52:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:36 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:36 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:36.877 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:36] "GET /metrics HTTP/1.1" 200 36178 "" "Prometheus/2.51.0" 2024-09-14T03:52:37.475 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-14T03:52:37.528 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:38.269 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:38.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:38.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:38 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:38.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:38 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:40 smithi001 ceph-mon[107399]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:40 smithi001 ceph-mon[112165]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:40 smithi012 ceph-mon[33640]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:40.900 INFO:teuthology.orchestra.run.smithi001.stdout:wait for majority of mons to upgrade 2024-09-14T03:52:41.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-14T03:52:41.803 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:42.120 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_2' created 2024-09-14T03:52:42.443 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:42 smithi001 ceph-mon[112165]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:42.444 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T03:52:42.444 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:42 smithi001 ceph-mon[107399]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:42.444 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T03:52:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:42 smithi012 ceph-mon[33640]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:42 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T03:52:43.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-14T03:52:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:43 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T03:52:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:43 smithi001 ceph-mon[107399]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:52:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:43 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T03:52:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:43 smithi001 ceph-mon[112165]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:52:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:43 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/636246234' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T03:52:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:43 smithi012 ceph-mon[33640]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:52:43.659 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[107399]: pgmap v62: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[107399]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[112165]: pgmap v62: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:44.172 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:44 smithi001 ceph-mon[112165]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:52:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:44 smithi012 ceph-mon[33640]: pgmap v62: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:44 smithi012 ceph-mon[33640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:52:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:44 smithi012 ceph-mon[33640]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:52:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:45 smithi001 ceph-mon[107399]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:52:45.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:45 smithi001 ceph-mon[112165]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:52:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:45 smithi012 ceph-mon[33640]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:52:46.390 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:46 smithi001 ceph-mon[107399]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:46.390 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:46 smithi001 ceph-mon[107399]: pgmap v65: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:46.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:46 smithi001 ceph-mon[112165]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:46.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:46 smithi001 ceph-mon[112165]: pgmap v65: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:46 smithi012 ceph-mon[33640]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:46 smithi012 ceph-mon[33640]: pgmap v65: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:46] "GET /metrics HTTP/1.1" 200 36178 "" "Prometheus/2.51.0" 2024-09-14T03:52:47.143 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-14T03:52:47.212 INFO:teuthology.orchestra.run.smithi001.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:47 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: pgmap v66: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:48.377 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: pgmap v66: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:48.378 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:48 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: pgmap v66: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='client.? 172.21.15.1:0/1075801985' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:48 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:48.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:52:49.084 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-14T03:52:49.087 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T03:52:49.087 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T03:52:49.138 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T03:52:49.138 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T03:52:49.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_117782 2024-09-14T03:52:49.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:52:49.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T03:52:49.424 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:49 smithi001 ceph-mon[112165]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.424 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:49 smithi001 ceph-mon[112165]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:49 smithi001 ceph-mon[107399]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:49 smithi001 ceph-mon[107399]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:49 smithi012 ceph-mon[33640]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:49 smithi012 ceph-mon[33640]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:52:49.996 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:50.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3062 3046 190.363 190.375 0.00861436 0.00522581 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[112165]: pgmap v68: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[107399]: pgmap v68: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T03:52:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:50 smithi012 ceph-mon[33640]: pgmap v68: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:52:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:50 smithi012 ceph-mon[33640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:52:50.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:50 smithi012 ceph-mon[33640]: Cluster is now healthy 2024-09-14T03:52:50.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:50 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:51.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 6109 6093 190.391 190.438 0.00440712 0.00523819 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: Upgrade: Setting container_image for all mgr 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.835 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:52:51.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:52:51.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:52:51.836 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:51 smithi012 ceph-mon[33640]: pgmap v69: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 419 op/s 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:52:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all mgr 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[107399]: pgmap v69: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 419 op/s 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all mgr 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:52:51.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:51 smithi001 ceph-mon[112165]: pgmap v69: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 419 op/s 2024-09-14T03:52:52.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 9320 9304 193.817 200.688 0.00306017 0.00514988 2024-09-14T03:52:53.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 12284 12268 191.671 185.25 0.0027909 0.00520868 2024-09-14T03:52:53.365 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: Upgrade: Updating mon.b 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: Deploying daemon mon.b on smithi012 2024-09-14T03:52:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[107399]: pgmap v70: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: Upgrade: Updating mon.b 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: Deploying daemon mon.b on smithi012 2024-09-14T03:52:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:53 smithi001 ceph-mon[112165]: pgmap v70: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s 2024-09-14T03:52:54.078 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: Upgrade: Updating mon.b 2024-09-14T03:52:54.078 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: from='mgr.34104 ' entity='mgr.y' 2024-09-14T03:52:54.078 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:52:54.078 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:52:54.079 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:54.079 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: Deploying daemon mon.b on smithi012 2024-09-14T03:52:54.079 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:53 smithi012 ceph-mon[33640]: pgmap v70: 89 pgs: 89 active+clean; 209 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s 2024-09-14T03:52:54.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 14960 14944 186.783 167.25 0.00907579 0.00534512 2024-09-14T03:52:54.609 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:52:54.609 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (3m) 17s ago 11m 20.9M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (3m) 19s ago 11m 78.9M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (10m) 17s ago 10m 21.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (10m) 19s ago 10m 15.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (5m) 19s ago 15m 454M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (5m) 17s ago 16m 511M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (2m) 17s ago 17m 43.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (15m) 19s ago 15m 60.9M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 8eff7b43abc7 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (102s) 17s ago 15m 35.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (4m) 17s ago 10m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (4m) 19s ago 10m 10.7M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (14m) 17s ago 14m 408M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:52:54.610 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (14m) 17s ago 14m 352M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (14m) 17s ago 14m 495M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (13m) 17s ago 13m 511M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (13m) 19s ago 13m 443M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (13m) 19s ago 12m 467M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (12m) 19s ago 12m 325M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (12m) 19s ago 12m 413M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:52:54.611 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (4m) 19s ago 11m 45.4M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:52:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:54 smithi001 ceph-mon[112165]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:54 smithi001 ceph-mon[112165]: from='client.25012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:54 smithi001 ceph-mon[107399]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:54 smithi001 ceph-mon[107399]: from='client.25012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:54 smithi012 ceph-mon[33640]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:54 smithi012 ceph-mon[33640]: from='client.25012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:54 smithi012 systemd[1]: Stopping Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:52:55.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 17460 17444 181.692 156.25 0.00781495 0.00549807 2024-09-14T03:52:55.336 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:52:55.336 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:52:55.336 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 1, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 11, 2024-09-14T03:52:55.337 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:52:55.338 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:52:55.338 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:52:55.710 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b[33617]: 2024-09-14T03:52:55.368+0000 7f24a8d76640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:52:55.710 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b[33617]: 2024-09-14T03:52:55.368+0000 7f24a8d76640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:52:55.710 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:55 smithi012 podman[101000]: 2024-09-14 03:52:55.410078833 +0000 UTC m=+0.403727237 container died 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:52:55.714 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE Bus STOPPING 2024-09-14T03:52:55.951 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/3 daemons upgraded", 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:52:55.952 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:52:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:52:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE Bus STOPPED 2024-09-14T03:52:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE Bus STARTING 2024-09-14T03:52:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE Serving on http://:::9283 2024-09-14T03:52:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:03:52:55] ENGINE Bus STARTED 2024-09-14T03:52:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:55 smithi012 podman[101000]: 2024-09-14 03:52:55.899421276 +0000 UTC m=+0.893069674 container cleanup 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-14T03:52:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:55 smithi012 bash[101000]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b 2024-09-14T03:52:56.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 15 19906 19891 177.582 152.938 0.00431025 0.00562491 2024-09-14T03:52:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 podman[101015]: 2024-09-14 03:52:56.201593196 +0000 UTC m=+0.784318245 container remove 8eff7b43abc736359ee05e8b207a43f199417756975987f979b43a7b9e1f6e5c (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:52:56.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-14T03:52:56.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 systemd[1]: Stopped Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:52:56.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.b.service: Consumed 12.926s CPU time. 2024-09-14T03:52:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:52:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:52:56] "GET /metrics HTTP/1.1" 200 36119 "" "Prometheus/2.51.0" 2024-09-14T03:52:56.987 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 systemd[1]: Starting Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:52:57.139 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 22532 22516 175.89 164.062 0.00660077 0.00567921 2024-09-14T03:52:57.266 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:56 smithi012 podman[101143]: 2024-09-14 03:52:56.892620007 +0000 UTC m=+0.027548150 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:52:57.267 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 podman[101143]: 2024-09-14 03:52:57.028340116 +0000 UTC m=+0.163268262 container create f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T03:52:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 podman[101143]: 2024-09-14 03:52:57.265379986 +0000 UTC m=+0.400308133 container init f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 podman[101143]: 2024-09-14 03:52:57.272800601 +0000 UTC m=+0.407728742 container start f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: pidfile_write: ignore empty --pid-file 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: load: jerasure load: lrc 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Git sha 0 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: DB SUMMARY 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: DB Session ID: OB2QB8YLSPSU9JORYAY0 2024-09-14T03:52:57.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1654 Bytes 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000030.sst 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000028.log size: 120003 ; 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.env: 0x560ea92de120 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:52:57.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.info_log: 0x560eaba61700 2024-09-14T03:52:57.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:52:57.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.statistics: (nil) 2024-09-14T03:52:57.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.use_fsync: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:52:57.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.db_log_dir: 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.wal_dir: 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:52:57.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.write_buffer_manager: 0x560eaba65b80 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:52:57.602 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.unordered_write: 0 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.row_cache: None 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.wal_filter: None 2024-09-14T03:52:57.603 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.wal_compression: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:52:57.604 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:52:57.605 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:52:57.605 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:52:57.605 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:52:57.605 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:52:57.606 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_open_files: -1 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Compression algorithms supported: 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kZSTD supported: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:52:57.607 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.merge_operator: 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_filter: None 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:52:57.608 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560eaba61320) 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_type: 0 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: data_block_index_type: 0 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_shortening: 1 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: checksum: 4 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: no_block_cache: 0 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache: 0x560eaba55610 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_options: 2024-09-14T03:52:57.609 INFO:journalctl@ceph.mon.b.smithi012.stdout: capacity : 536870912 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: num_shard_bits : 4 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: strict_capacity_limit : 0 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_cache_compressed: (nil) 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: persistent_cache: (nil) 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_size: 4096 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_size_deviation: 10 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_restart_interval: 16 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: index_block_restart_interval: 1 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: metadata_block_size: 4096 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: partition_filters: 0 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: use_delta_encoding: 1 2024-09-14T03:52:57.610 INFO:journalctl@ceph.mon.b.smithi012.stdout: filter_policy: bloomfilter 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: whole_key_filtering: 1 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: verify_compression: 0 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: format_version: 5 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: enable_index_compression: 1 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: block_align: 0 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: max_auto_readahead_size: 262144 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: prepopulate_block_cache: 0 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:52:57.611 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression: NoCompression 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.num_levels: 7 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:52:57.612 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:52:57.613 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:52:57.614 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:52:57.615 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:52:57.616 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:52:57.617 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.ttl: 2592000 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:52:57.618 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 32, last_sequence is 18115, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 28 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e2f3c8a5-7108-4746-90b3-36ffe31dde19 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285977316241, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285977317488, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 79541, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18120, "largest_seqno": 18247, "table_properties": {"data_size": 78010, "index_size": 301, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 197, "raw_key_size": 1956, "raw_average_key_size": 26, "raw_value_size": 76390, "raw_average_value_size": 1032, "num_data_blocks": 14, "num_entries": 74, "num_filter_entries": 74, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285977, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e2f3c8a5-7108-4746-90b3-36ffe31dde19", "db_session_id": "OB2QB8YLSPSU9JORYAY0", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285977317653, "job": 1, "event": "recovery_finished"} 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560eaba84e00 2024-09-14T03:52:57.619 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: DB pointer 0x560eabb86000 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** DB Stats ** 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-14T03:52:57.620 INFO:journalctl@ceph.mon.b.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: L0 1/0 77.68 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: L6 1/0 10.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: Sum 2/0 10.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 103.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 103.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:52:57.621 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Cumulative compaction: 0.00 GB write, 15.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Interval compaction: 0.00 GB write, 15.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Block cache BinnedLRUCache@0x560eaba55610#7 capacity: 512.00 MB usage: 0.62 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.23 KB,4.47035e-05%) IndexBlock(1,0.39 KB,7.45058e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:52:57.622 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: starting mon.b rank 1 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???) e3 preinit fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).mds e9 new map 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).mds e9 print_map 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: e9 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: legacy client fscid: 1 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: Filesystem 'foo' (1) 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: fs_name foo 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: epoch 8 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:52:57.623 INFO:journalctl@ceph.mon.b.smithi012.stdout: created 2024-09-14T03:42:34.635441+0000 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: modified 2024-09-14T03:42:47.984273+0000 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: tableserver 0 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: root 0 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: session_timeout 60 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: session_autoclose 300 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: max_file_size 1099511627776 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: max_xattr_size 65536 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: required_client_features {} 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: last_failure 0 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: last_failure_osd_epoch 0 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: max_mds 1 2024-09-14T03:52:57.624 INFO:journalctl@ceph.mon.b.smithi012.stdout: in 0 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: up {0=24557} 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: failed 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: damaged 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: stopped 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: data_pools [4] 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: metadata_pool 3 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: inline_data disabled 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: balancer 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: bal_rank_mask -1 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: standby_count_wanted 1 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: [mds.foo.smithi001.ytjgbj{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.1:6834/2296041448,v1:172.21.15.1:6835/2296041448] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.625 INFO:journalctl@ceph.mon.b.smithi012.stdout: Standby daemons: 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout: 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout: [mds.foo.smithi012.nnhswt{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.12:6832/803290127,v1:172.21.15.12:6833/803290127] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 ceph-mon[101180]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 bash[101143]: f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f 2024-09-14T03:52:57.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:57 smithi012 systemd[1]: Started Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:52:58.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 25060 25044 173.901 158 0.00639471 0.00574394 2024-09-14T03:52:59.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 27446 27430 171.422 149.125 0.00693242 0.00582743 2024-09-14T03:52:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: mon.a calling monitor election 2024-09-14T03:52:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: mon.b calling monitor election 2024-09-14T03:52:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: mon.c calling monitor election 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: monmap epoch 4 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: last_changed 2024-09-14T03:52:58.692763+0000 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: min_mon_release 19 (squid) 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: election_strategy: 1 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:52:59.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: overall HEALTH_OK 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[107399]: pgmap v73: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: mon.a calling monitor election 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: mon.b calling monitor election 2024-09-14T03:52:59.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: mon.c calling monitor election 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: monmap epoch 4 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: last_changed 2024-09-14T03:52:58.692763+0000 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: min_mon_release 19 (squid) 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: election_strategy: 1 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:52:59.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: overall HEALTH_OK 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:59.975 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:52:59 smithi001 ceph-mon[112165]: pgmap v73: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: mon.a calling monitor election 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: mon.b calling monitor election 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: mon.c calling monitor election 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: monmap epoch 4 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: last_changed 2024-09-14T03:52:58.692763+0000 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: created 2024-09-14T03:35:47.568727+0000 2024-09-14T03:53:00.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: min_mon_release 19 (squid) 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: election_strategy: 1 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: 0: [v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] mon.a 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: 1: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.b 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: 2: [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0] mon.c 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: fsmap foo:1 {0=foo.smithi001.ytjgbj=up:active} 1 up:standby 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: overall HEALTH_OK 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:52:59 smithi012 ceph-mon[101180]: pgmap v73: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:53:00.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 29769 29753 169.036 145.188 0.00424338 0.00591041 2024-09-14T03:53:01.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 31826 31810 165.662 128.562 0.00673352 0.00602912 2024-09-14T03:53:02.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 33790 33774 162.361 122.75 0.00794375 0.0061527 2024-09-14T03:53:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:02 smithi012 ceph-mon[101180]: pgmap v74: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-14T03:53:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:02 smithi001 ceph-mon[107399]: pgmap v74: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-14T03:53:02.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:02 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:53:02.339+0000 7fc9b3f23640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T03:53:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:02 smithi001 ceph-mon[112165]: pgmap v74: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-14T03:53:03.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 35919 35903 160.267 133.062 0.00651473 0.00623436 2024-09-14T03:53:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:03 smithi001 ceph-mon[107399]: pgmap v75: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-14T03:53:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:03 smithi001 ceph-mon[112165]: pgmap v75: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-14T03:53:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:03 smithi012 ceph-mon[101180]: pgmap v75: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-14T03:53:04.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 37893 37877 157.807 123.375 0.00725007 0.00633179 2024-09-14T03:53:05.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 39830 39814 155.509 121.062 0.00739905 0.00642456 2024-09-14T03:53:05.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:06.140 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 41790 41774 153.567 122.5 0.00452865 0.00650679 2024-09-14T03:53:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:05 smithi001 ceph-mon[107399]: pgmap v76: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:53:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:05 smithi001 ceph-mon[112165]: pgmap v76: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:53:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:05 smithi012 ceph-mon[101180]: pgmap v76: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:53:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:06] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.51.0" 2024-09-14T03:53:07.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 43572 43556 151.223 111.375 0.00341246 0.00660716 2024-09-14T03:53:08.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 45333 45317 149.056 110.062 0.00485677 0.00670302 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[107399]: pgmap v77: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[112165]: pgmap v77: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:08 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:08 smithi012 ceph-mon[101180]: pgmap v77: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-09-14T03:53:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:08 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:08 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:53:09.140486+0000 min lat: 0.0014891 max lat: 0.0803904 avg lat: 0.00678556 2024-09-14T03:53:09.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:53:09.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 47142 47126 147.255 113.062 0.00643099 0.00678556 2024-09-14T03:53:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:09 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:09 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:09 smithi012 ceph-mon[101180]: pgmap v78: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[107399]: pgmap v78: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:09 smithi001 ceph-mon[112165]: pgmap v78: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:53:10.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 48924 48908 145.546 111.375 0.00592173 0.00686548 2024-09-14T03:53:11.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 50564 50548 143.589 102.5 0.00970038 0.00695858 2024-09-14T03:53:12.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 52285 52269 142.022 107.562 0.00898025 0.00703606 2024-09-14T03:53:12.328 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:12 smithi012 ceph-mon[101180]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 163 MiB/s wr, 2.62k op/s 2024-09-14T03:53:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:12 smithi001 ceph-mon[112165]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 163 MiB/s wr, 2.62k op/s 2024-09-14T03:53:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:12 smithi001 ceph-mon[107399]: pgmap v79: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 163 MiB/s wr, 2.62k op/s 2024-09-14T03:53:13.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 53811 53795 140.079 95.375 0.0167836 0.00713339 2024-09-14T03:53:14.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 55266 55250 138.113 90.9375 0.0146785 0.00723586 2024-09-14T03:53:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: pgmap v80: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-14T03:53:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: pgmap v80: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:14.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:14.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:14.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: pgmap v80: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:15.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 56739 56723 136.341 92.0625 0.00966582 0.00733019 2024-09-14T03:53:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:15 smithi001 ceph-mon[107399]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:53:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:15 smithi001 ceph-mon[107399]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:53:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:15 smithi001 ceph-mon[112165]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:53:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:15 smithi001 ceph-mon[112165]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:53:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:15 smithi012 ceph-mon[101180]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:53:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:15 smithi012 ceph-mon[101180]: Reconfiguring daemon mon.a on smithi001 2024-09-14T03:53:16.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 58307 58291 134.921 98 0.0114663 0.00740593 2024-09-14T03:53:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:16 smithi001 ceph-mon[107399]: pgmap v81: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:53:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:16 smithi001 ceph-mon[112165]: pgmap v81: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:53:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:16 smithi012 ceph-mon[101180]: pgmap v81: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T03:53:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:16] "GET /metrics HTTP/1.1" 200 36186 "" "Prometheus/2.51.0" 2024-09-14T03:53:17.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 59731 59715 133.28 89 0.0122622 0.00749684 2024-09-14T03:53:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:53:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:53:17.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[107399]: pgmap v82: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-14T03:53:17.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:53:17.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:17 smithi001 ceph-mon[112165]: pgmap v82: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:17.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: Reconfiguring daemon mgr.y on smithi001 2024-09-14T03:53:17.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:17 smithi012 ceph-mon[101180]: pgmap v82: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-14T03:53:18.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 61274 61258 132.01 96.4375 0.0176872 0.0075698 2024-09-14T03:53:19.141 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 62657 62641 130.49 86.4375 0.0114777 0.00765856 2024-09-14T03:53:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:53:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[107399]: pgmap v83: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:19.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:19.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:53:19.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:19 smithi001 ceph-mon[112165]: pgmap v83: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T03:53:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:53:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: Reconfiguring daemon mon.c on smithi001 2024-09-14T03:53:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:19 smithi012 ceph-mon[101180]: pgmap v83: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T03:53:20.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 63933 63917 128.853 79.75 0.0227905 0.00775531 2024-09-14T03:53:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:53:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:53:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:21.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 65074 65058 127.055 71.3125 0.0163814 0.00786519 2024-09-14T03:53:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[107399]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:53:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.0 on smithi001 2024-09-14T03:53:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[107399]: pgmap v84: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:53:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[112165]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:53:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.0 on smithi001 2024-09-14T03:53:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:21 smithi001 ceph-mon[112165]: pgmap v84: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:53:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:21 smithi012 ceph-mon[101180]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:53:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:21 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.0 on smithi001 2024-09-14T03:53:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:21 smithi012 ceph-mon[101180]: pgmap v84: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T03:53:22.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 66358 66342 125.636 80.25 0.00669447 0.00795303 2024-09-14T03:53:23.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 67217 67201 123.52 53.6875 0.0133402 0.00809048 2024-09-14T03:53:24.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 68569 68553 122.405 84.5 0.00664592 0.00816406 2024-09-14T03:53:24.454 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:24 smithi001 ceph-mon[107399]: pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:53:24.455 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:24 smithi001 ceph-mon[112165]: pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:53:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:24 smithi012 ceph-mon[101180]: pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T03:53:25.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 69852 69836 121.232 80.1875 0.0206595 0.0082433 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.1 on smithi001 2024-09-14T03:53:25.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[107399]: pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.1 on smithi001 2024-09-14T03:53:25.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:25 smithi001 ceph-mon[112165]: pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.1 on smithi001 2024-09-14T03:53:26.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:25 smithi012 ceph-mon[101180]: pgmap v86: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-14T03:53:26.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 71166 71150 120.175 82.125 0.00762011 0.00831532 2024-09-14T03:53:26.578 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:53:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:26] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.51.0" 2024-09-14T03:53:27.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 72226 72210 118.756 66.25 0.0277076 0.00841436 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.2 on smithi001 2024-09-14T03:53:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[107399]: pgmap v87: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.2 on smithi001 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:27.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:27 smithi001 ceph-mon[112165]: pgmap v87: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (4m) 50s ago 11m 20.9M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (3m) 19s ago 11m 81.2M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (10m) 50s ago 10m 21.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (10m) 19s ago 10m 15.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (5m) 19s ago 15m 455M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (6m) 50s ago 17m 511M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (2m) 50s ago 17m 43.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (30s) 19s ago 16m 19.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:53:27.741 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (2m) 50s ago 16m 35.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (5m) 50s ago 11m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (5m) 19s ago 11m 10.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (15m) 50s ago 15m 408M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (15m) 50s ago 15m 352M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (14m) 50s ago 14m 495M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (14m) 50s ago 14m 511M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (13m) 19s ago 13m 420M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (13m) 19s ago 13m 433M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (13m) 19s ago 13m 314M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (12m) 19s ago 12m 382M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:53:27.742 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (4m) 19s ago 12m 45.4M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.2 on smithi001 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:27 smithi012 ceph-mon[101180]: pgmap v87: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-14T03:53:28.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 73373 73357 117.549 71.6875 0.00726245 0.0085019 2024-09-14T03:53:28.420 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:53:28.420 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:53:28.420 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 10, 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:53:28.421 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:53:28.422 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:53:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:28 smithi012 ceph-mon[101180]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:28 smithi012 ceph-mon[101180]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2224264888' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[107399]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[107399]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2224264888' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[112165]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[112165]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:28.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2224264888' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:29.012 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:53:29.012 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:53:29.012 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [ 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "mon" 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "3/3 daemons upgraded", 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:53:29.013 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:53:29.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:53:29.142276+0000 min lat: 0.0014891 max lat: 0.1333 avg lat: 0.0085983 2024-09-14T03:53:29.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:53:29.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 74409 74393 116.229 64.75 0.0123698 0.0085983 2024-09-14T03:53:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:53:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.3 on smithi001 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[107399]: pgmap v88: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.3 on smithi001 2024-09-14T03:53:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:29.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:29 smithi001 ceph-mon[112165]: pgmap v88: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.3 on smithi001 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:29 smithi012 ceph-mon[101180]: pgmap v88: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:30.142 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 75664 75648 115.307 78.4375 0.00695416 0.0086674 2024-09-14T03:53:31.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 76858 76842 114.338 74.625 0.0137004 0.00873983 2024-09-14T03:53:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: Reconfiguring mds.foo.smithi001.ytjgbj (monmap changed)... 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: Reconfiguring daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:53:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:31 smithi012 ceph-mon[101180]: pgmap v89: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-14T03:53:32.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 78050 78034 113.411 74.5 0.00937228 0.00881144 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: Reconfiguring mds.foo.smithi001.ytjgbj (monmap changed)... 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: Reconfiguring daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:53:32.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[107399]: pgmap v89: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: Reconfiguring mds.foo.smithi001.ytjgbj (monmap changed)... 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi001.ytjgbj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: Reconfiguring daemon mds.foo.smithi001.ytjgbj on smithi001 2024-09-14T03:53:32.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:31 smithi001 ceph-mon[112165]: pgmap v89: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-14T03:53:33.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 79297 79281 112.605 77.9375 0.00419918 0.00887543 2024-09-14T03:53:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:53:33.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:33 smithi012 ceph-mon[101180]: pgmap v90: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:53:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[107399]: pgmap v90: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: Reconfiguring daemon mon.b on smithi012 2024-09-14T03:53:33.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:33 smithi001 ceph-mon[112165]: pgmap v90: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T03:53:34.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 80551 80535 111.844 78.375 0.00881685 0.0089359 2024-09-14T03:53:35.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 81739 81723 111.027 74.25 0.010063 0.00900157 2024-09-14T03:53:35.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:36.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 82827 82811 110.111 68 0.0103587 0.00907613 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: Reconfiguring daemon mgr.x on smithi012 2024-09-14T03:53:36.241 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:35 smithi012 ceph-mon[101180]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: Reconfiguring daemon mgr.x on smithi012 2024-09-14T03:53:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[107399]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: Reconfiguring daemon mgr.x on smithi012 2024-09-14T03:53:36.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:35 smithi001 ceph-mon[112165]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T03:53:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:36] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-09-14T03:53:37.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 84051 84035 109.411 76.5 0.015176 0.00913481 2024-09-14T03:53:38.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 85275 85259 108.739 76.5 0.010902 0.00919105 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[107399]: pgmap v92: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[112165]: pgmap v92: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:53:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:38.478 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:38 smithi012 ceph-mon[101180]: pgmap v92: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:38.478 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.478 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:38.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:53:38.479 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:39.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 86470 86454 108.058 74.6875 0.0144481 0.00924932 2024-09-14T03:53:39.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:39 smithi001 ceph-mon[107399]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:53:39.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:39 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.4 on smithi012 2024-09-14T03:53:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:39 smithi001 ceph-mon[112165]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:53:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:39 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.4 on smithi012 2024-09-14T03:53:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:39 smithi012 ceph-mon[101180]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:53:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:39 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.4 on smithi012 2024-09-14T03:53:40.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 87639 87623 107.371 73.0625 0.0154007 0.00930818 2024-09-14T03:53:40.433 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:40 smithi012 ceph-mon[101180]: pgmap v93: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T03:53:40.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:53:40.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[107399]: pgmap v93: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T03:53:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[112165]: pgmap v93: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:53:40.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:41.143 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 88808 88792 106.712 73.0625 0.0298968 0.00936521 2024-09-14T03:53:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:41 smithi001 ceph-mon[107399]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:53:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:41 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.5 on smithi012 2024-09-14T03:53:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:41 smithi001 ceph-mon[112165]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:53:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:41 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.5 on smithi012 2024-09-14T03:53:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:41 smithi012 ceph-mon[101180]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:53:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:41 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.5 on smithi012 2024-09-14T03:53:42.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 89944 89928 106.038 71 0.0166689 0.00942533 2024-09-14T03:53:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:42 smithi012 ceph-mon[101180]: pgmap v94: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:53:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:42 smithi001 ceph-mon[107399]: pgmap v94: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:53:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:42 smithi001 ceph-mon[112165]: pgmap v94: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T03:53:43.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 15 91076 91061 105.385 70.8125 0.00537516 0.00948303 2024-09-14T03:53:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:53:43.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:53:43.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:43.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.6 on smithi012 2024-09-14T03:53:43.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:43 smithi012 ceph-mon[101180]: pgmap v95: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-09-14T03:53:44.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 92205 92189 104.751 70.5 0.0100596 0.00954086 2024-09-14T03:53:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:53:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:53:44.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:44.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.6 on smithi012 2024-09-14T03:53:44.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[107399]: pgmap v95: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-09-14T03:53:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:44.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:53:44.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:53:44.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:44.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.6 on smithi012 2024-09-14T03:53:44.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:43 smithi001 ceph-mon[112165]: pgmap v95: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-09-14T03:53:45.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 15 93324 93309 104.13 70 0.0175122 0.00959795 2024-09-14T03:53:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:53:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:53:45.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:45.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: Reconfiguring daemon osd.7 on smithi012 2024-09-14T03:53:45.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:45 smithi012 ceph-mon[101180]: pgmap v96: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: Reconfiguring daemon osd.7 on smithi012 2024-09-14T03:53:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[107399]: pgmap v96: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: Reconfiguring daemon osd.7 on smithi012 2024-09-14T03:53:45.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:45 smithi001 ceph-mon[112165]: pgmap v96: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:53:46.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 94437 94421 103.522 69.5 0.0318647 0.00965462 2024-09-14T03:53:46.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:46] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-09-14T03:53:47.247 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 95526 95510 102.911 68.0625 0.0123813 0.00971163 2024-09-14T03:53:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: Reconfiguring mds.foo.smithi012.nnhswt (monmap changed)... 2024-09-14T03:53:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:47.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:47.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: Reconfiguring daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:53:47.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:47 smithi012 ceph-mon[101180]: pgmap v97: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: Reconfiguring mds.foo.smithi012.nnhswt (monmap changed)... 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: Reconfiguring daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:53:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[107399]: pgmap v97: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: Reconfiguring mds.foo.smithi012.nnhswt (monmap changed)... 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.nnhswt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: Reconfiguring daemon mds.foo.smithi012.nnhswt on smithi012 2024-09-14T03:53:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:47 smithi001 ceph-mon[112165]: pgmap v97: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T03:53:48.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 96624 96608 102.33 68.625 0.0132129 0.00976674 2024-09-14T03:53:49.144 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:53:49.144108+0000 min lat: 0.0014891 max lat: 0.1333 avg lat: 0.00983554 2024-09-14T03:53:49.145 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:53:49.145 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 97578 97562 101.618 59.625 0.0240433 0.00983554 2024-09-14T03:53:49.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:49.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:53:49.410 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:49.412 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:53:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.416 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:53:49.416 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:53:49.416 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:53:49.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:53:49.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:50.147 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 98434 98418 100.825 53.5 0.0179794 0.00991246 2024-09-14T03:53:50.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:53:50.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:53:50.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:53:50.411 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all grafana 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all loki 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all promtail 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Finalizing container_image settings 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: Upgrade: Complete! 2024-09-14T03:53:50.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: pgmap v98: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:53:50.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all grafana 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all loki 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all promtail 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Finalizing container_image settings 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: Upgrade: Complete! 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: pgmap v98: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:50.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:53:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:53:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all grafana 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all loki 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all promtail 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Finalizing container_image settings 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: Upgrade: Complete! 2024-09-14T03:53:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: pgmap v98: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2024-09-14T03:53:50.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:50.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:51.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 99280 99264 100.05 52.875 0.0288817 0.00998798 2024-09-14T03:53:52.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 100243 100227 99.4172 60.1875 0.0199942 0.0100529 2024-09-14T03:53:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:52 smithi001 ceph-mon[112165]: pgmap v99: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:52.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:52 smithi001 ceph-mon[107399]: pgmap v99: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:52 smithi012 ceph-mon[101180]: pgmap v99: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T03:53:53.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 101259 101243 98.856 63.5 0.0282472 0.0101108 2024-09-14T03:53:54.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 102171 102155 98.212 57 0.0151579 0.0101761 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: pgmap v100: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: pgmap v100: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: pgmap v100: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:55.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 103183 103167 97.6822 63.25 0.0138541 0.0102319 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:53:56.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 104219 104203 97.1906 64.75 0.0130251 0.0102834 2024-09-14T03:53:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:56 smithi001 ceph-mon[107399]: pgmap v101: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-14T03:53:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:56 smithi001 ceph-mon[112165]: pgmap v101: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-14T03:53:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:56 smithi012 ceph-mon[101180]: pgmap v101: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2024-09-14T03:53:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:53:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:53:56] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.51.0" 2024-09-14T03:53:57.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 105246 105230 96.7052 64.1875 0.0296872 0.0103352 2024-09-14T03:53:58.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 106250 106234 96.213 62.75 0.0112304 0.010388 2024-09-14T03:53:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:53:58 smithi001 ceph-mon[112165]: pgmap v102: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:53:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:53:58 smithi001 ceph-mon[107399]: pgmap v102: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:53:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:53:58 smithi012 ceph-mon[101180]: pgmap v102: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:53:59.148 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 15 107251 107236 95.7332 62.625 0.0200995 0.0104401 2024-09-14T03:54:00.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 108181 108165 95.2025 58.0625 0.00850987 0.0104978 2024-09-14T03:54:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:00 smithi001 ceph-mon[107399]: pgmap v103: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:54:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:00 smithi001 ceph-mon[112165]: pgmap v103: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:54:00.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T03:54:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:00 smithi012 ceph-mon[101180]: pgmap v103: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-09-14T03:54:00.903 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:01.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 109100 109084 94.6775 57.4375 0.0142532 0.0105567 2024-09-14T03:54:01.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:01 smithi001 ceph-mon[107399]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:01.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:01 smithi001 ceph-mon[112165]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:01 smithi012 ceph-mon[101180]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:02.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 110049 110033 94.193 59.3125 0.00781618 0.010611 2024-09-14T03:54:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:02 smithi012 ceph-mon[101180]: pgmap v104: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T03:54:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:02 smithi001 ceph-mon[112165]: pgmap v104: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T03:54:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:02 smithi001 ceph-mon[107399]: pgmap v104: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T03:54:03.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 110891 110875 93.6312 52.625 0.0198153 0.0106749 2024-09-14T03:54:03.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:03 smithi001 ceph-mon[107399]: pgmap v105: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2024-09-14T03:54:03.430 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:03 smithi001 ceph-mon[112165]: pgmap v105: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2024-09-14T03:54:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:03 smithi012 ceph-mon[101180]: pgmap v105: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2024-09-14T03:54:04.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 111859 111843 93.1894 60.5 0.0153737 0.010725 2024-09-14T03:54:05.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 112815 112799 92.7494 59.75 0.0175909 0.0107759 2024-09-14T03:54:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:05.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:06.149 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 15 113771 113756 92.3216 59.8125 0.0131189 0.0108255 2024-09-14T03:54:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:05 smithi001 ceph-mon[112165]: pgmap v106: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T03:54:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:05 smithi001 ceph-mon[107399]: pgmap v106: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T03:54:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:05 smithi012 ceph-mon[101180]: pgmap v106: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T03:54:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:06] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-09-14T03:54:07.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 114737 114721 91.9111 60.3125 0.0179346 0.0108747 2024-09-14T03:54:08.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 115679 115663 91.4929 58.875 0.0329035 0.0109242 2024-09-14T03:54:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:08 smithi001 ceph-mon[107399]: pgmap v107: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T03:54:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:08 smithi001 ceph-mon[112165]: pgmap v107: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T03:54:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:08 smithi012 ceph-mon[101180]: pgmap v107: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T03:54:09.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:54:09.149675+0000 min lat: 0.0014891 max lat: 0.151685 avg lat: 0.0109737 2024-09-14T03:54:09.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:54:09.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 116615 116599 91.0805 58.5 0.0135109 0.0109737 2024-09-14T03:54:10.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 117550 117534 90.6774 58.4375 0.0124461 0.0110227 2024-09-14T03:54:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:10 smithi001 ceph-mon[107399]: pgmap v108: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-09-14T03:54:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:10 smithi001 ceph-mon[112165]: pgmap v108: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-09-14T03:54:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:10 smithi012 ceph-mon[101180]: pgmap v108: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-09-14T03:54:11.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 118474 118458 90.2758 57.75 0.0151633 0.011071 2024-09-14T03:54:12.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 119396 119380 89.8824 57.625 0.0182963 0.0111203 2024-09-14T03:54:12.439 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:12 smithi012 ceph-mon[101180]: pgmap v109: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:54:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:12 smithi001 ceph-mon[107399]: pgmap v109: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:54:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:12 smithi001 ceph-mon[112165]: pgmap v109: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T03:54:13.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 120315 120299 89.4961 57.4375 0.0133587 0.0111686 2024-09-14T03:54:14.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 121239 121223 89.1226 57.75 0.0105242 0.0112152 2024-09-14T03:54:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:14 smithi001 ceph-mon[107399]: pgmap v110: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T03:54:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:14 smithi001 ceph-mon[112165]: pgmap v110: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T03:54:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:14 smithi012 ceph-mon[101180]: pgmap v110: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T03:54:15.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 122150 122134 88.7483 56.9375 0.0215374 0.0112628 2024-09-14T03:54:16.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 123051 123035 88.3754 56.3125 0.0171019 0.0113103 2024-09-14T03:54:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:16 smithi001 ceph-mon[107399]: pgmap v111: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-14T03:54:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:16 smithi001 ceph-mon[112165]: pgmap v111: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-14T03:54:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:16 smithi012 ceph-mon[101180]: pgmap v111: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-09-14T03:54:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:16] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-09-14T03:54:17.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 123961 123945 88.0174 56.875 0.0144711 0.0113562 2024-09-14T03:54:18.150 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 124856 124840 87.6569 55.9375 0.0128492 0.0114026 2024-09-14T03:54:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:18 smithi001 ceph-mon[107399]: pgmap v112: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:54:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:18 smithi001 ceph-mon[112165]: pgmap v112: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:54:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:18 smithi012 ceph-mon[101180]: pgmap v112: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0089 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 125722 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 87.2983 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 37.0345 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 200.688 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 52.625 2024-09-14T03:54:19.151 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1396 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 592.552 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3211 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 842 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0114505 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.00776205 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 0.151685 2024-09-14T03:54:19.152 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.0014891 2024-09-14T03:54:19.158 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T03:54:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:19 smithi012 ceph-mon[101180]: pgmap v113: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2024-09-14T03:54:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:19 smithi001 ceph-mon[107399]: pgmap v113: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2024-09-14T03:54:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:19 smithi001 ceph-mon[112165]: pgmap v113: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2024-09-14T03:54:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:21 smithi012 ceph-mon[101180]: pgmap v114: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.15k op/s 2024-09-14T03:54:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:21 smithi001 ceph-mon[107399]: pgmap v114: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.15k op/s 2024-09-14T03:54:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:21 smithi001 ceph-mon[112165]: pgmap v114: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.15k op/s 2024-09-14T03:54:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:24 smithi001 ceph-mon[107399]: pgmap v115: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 748 op/s 2024-09-14T03:54:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:24 smithi001 ceph-mon[112165]: pgmap v115: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 748 op/s 2024-09-14T03:54:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:24 smithi012 ceph-mon[101180]: pgmap v115: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 748 op/s 2024-09-14T03:54:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:26 smithi001 ceph-mon[107399]: pgmap v116: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 819 op/s 2024-09-14T03:54:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:26 smithi001 ceph-mon[112165]: pgmap v116: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 819 op/s 2024-09-14T03:54:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:26 smithi012 ceph-mon[101180]: pgmap v116: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 819 op/s 2024-09-14T03:54:26.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:26] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.51.0" 2024-09-14T03:54:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:28 smithi001 ceph-mon[107399]: pgmap v117: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.05k op/s 2024-09-14T03:54:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:28 smithi001 ceph-mon[112165]: pgmap v117: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.05k op/s 2024-09-14T03:54:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:28 smithi012 ceph-mon[101180]: pgmap v117: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.05k op/s 2024-09-14T03:54:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:30 smithi001 ceph-mon[107399]: pgmap v118: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 744 op/s 2024-09-14T03:54:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:30 smithi001 ceph-mon[112165]: pgmap v118: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 744 op/s 2024-09-14T03:54:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:30 smithi012 ceph-mon[101180]: pgmap v118: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 744 op/s 2024-09-14T03:54:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:32 smithi012 ceph-mon[101180]: pgmap v119: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 1.12k op/s 2024-09-14T03:54:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:32 smithi001 ceph-mon[107399]: pgmap v119: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 1.12k op/s 2024-09-14T03:54:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:32 smithi001 ceph-mon[112165]: pgmap v119: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 1.12k op/s 2024-09-14T03:54:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:34 smithi001 ceph-mon[107399]: pgmap v120: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:34 smithi001 ceph-mon[112165]: pgmap v120: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:34 smithi012 ceph-mon[101180]: pgmap v120: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:36 smithi001 ceph-mon[107399]: pgmap v121: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:36 smithi001 ceph-mon[112165]: pgmap v121: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:36 smithi012 ceph-mon[101180]: pgmap v121: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-14T03:54:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:36] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.51.0" 2024-09-14T03:54:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:37 smithi001 ceph-mon[107399]: pgmap v122: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:54:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:37 smithi001 ceph-mon[112165]: pgmap v122: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:54:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:37 smithi012 ceph-mon[101180]: pgmap v122: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:54:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:40 smithi001 ceph-mon[107399]: pgmap v123: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2024-09-14T03:54:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:40 smithi001 ceph-mon[112165]: pgmap v123: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2024-09-14T03:54:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:40 smithi012 ceph-mon[101180]: pgmap v123: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2024-09-14T03:54:42.443 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:42 smithi012 ceph-mon[101180]: pgmap v124: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:54:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:42 smithi001 ceph-mon[107399]: pgmap v124: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:54:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:42 smithi001 ceph-mon[112165]: pgmap v124: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-14T03:54:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:44 smithi001 ceph-mon[107399]: pgmap v125: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:44 smithi001 ceph-mon[112165]: pgmap v125: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:44 smithi012 ceph-mon[101180]: pgmap v125: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:46 smithi001 ceph-mon[107399]: pgmap v126: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:46 smithi001 ceph-mon[112165]: pgmap v126: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:46 smithi012 ceph-mon[101180]: pgmap v126: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-09-14T03:54:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:46] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.51.0" 2024-09-14T03:54:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:48 smithi001 ceph-mon[107399]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:54:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:48 smithi001 ceph-mon[112165]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:54:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:48 smithi012 ceph-mon[101180]: pgmap v127: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T03:54:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:49 smithi012 ceph-mon[101180]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-14T03:54:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:49 smithi001 ceph-mon[107399]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-14T03:54:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:49 smithi001 ceph-mon[112165]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-14T03:54:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:51 smithi012 ceph-mon[101180]: pgmap v129: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:54:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:51 smithi001 ceph-mon[112165]: pgmap v129: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:54:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:51 smithi001 ceph-mon[107399]: pgmap v129: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T03:54:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:54 smithi001 ceph-mon[107399]: pgmap v130: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:54.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:54 smithi001 ceph-mon[112165]: pgmap v130: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:54 smithi012 ceph-mon[101180]: pgmap v130: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:56 smithi012 ceph-mon[101180]: pgmap v131: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:56 smithi001 ceph-mon[107399]: pgmap v131: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:56 smithi001 ceph-mon[112165]: pgmap v131: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-14T03:54:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:54:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:54:56] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-14T03:54:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:54:58 smithi012 ceph-mon[101180]: pgmap v132: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-14T03:54:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:54:58 smithi001 ceph-mon[107399]: pgmap v132: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-14T03:54:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:54:58 smithi001 ceph-mon[112165]: pgmap v132: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-14T03:55:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[112165]: pgmap v133: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-14T03:55:00.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[107399]: pgmap v133: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:00 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:00 smithi012 ceph-mon[101180]: pgmap v133: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-14T03:55:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:00 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:00 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:00 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:00 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:01.276 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:01 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.276 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:01 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.276 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:01 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:01 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:02 smithi001 ceph-mon[107399]: pgmap v134: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:02 smithi001 ceph-mon[112165]: pgmap v134: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:02 smithi012 ceph-mon[101180]: pgmap v134: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:03.444 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:03 smithi001 ceph-mon[107399]: pgmap v135: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:03.444 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:03 smithi001 ceph-mon[112165]: pgmap v135: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:03 smithi012 ceph-mon[101180]: pgmap v135: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:04.208 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for all mons to upgrade' 2024-09-14T03:55:04.582 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:05.115 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:05.115 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:06.149 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:05 smithi001 ceph-mon[112165]: pgmap v136: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:06.149 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:05 smithi001 ceph-mon[107399]: pgmap v136: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:05 smithi012 ceph-mon[101180]: pgmap v136: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:55:06.784 INFO:teuthology.orchestra.run.smithi001.stdout:wait for all mons to upgrade 2024-09-14T03:55:06.784 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:06] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.51.0" 2024-09-14T03:55:07.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-09-14T03:55:07.884 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:08.391 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:08 smithi001 ceph-mon[112165]: pgmap v137: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:55:08.392 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:08 smithi001 ceph-mon[107399]: pgmap v137: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:55:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:08 smithi012 ceph-mon[101180]: pgmap v137: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:55:10.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:10 smithi001 ceph-mon[112165]: pgmap v138: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-14T03:55:10.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:10 smithi001 ceph-mon[107399]: pgmap v138: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-14T03:55:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:10 smithi012 ceph-mon[101180]: pgmap v138: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-14T03:55:12.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:12 smithi012 ceph-mon[101180]: from='client.54121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:12.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:12 smithi012 ceph-mon[101180]: pgmap v139: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T03:55:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:12 smithi001 ceph-mon[112165]: from='client.54121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:12 smithi001 ceph-mon[112165]: pgmap v139: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T03:55:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:12 smithi001 ceph-mon[107399]: from='client.54121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:12 smithi001 ceph-mon[107399]: pgmap v139: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T03:55:12.973 INFO:teuthology.orchestra.run.smithi001.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:13.714 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:55:13.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: pgmap v140: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[107399]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:14.115 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: pgmap v140: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:13 smithi001 ceph-mon[112165]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:14.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: pgmap v140: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:14.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:13 smithi012 ceph-mon[101180]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:16 smithi001 ceph-mon[112165]: pgmap v141: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:16 smithi001 ceph-mon[107399]: pgmap v141: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:16 smithi012 ceph-mon[101180]: pgmap v141: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:55:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:16] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.51.0" 2024-09-14T03:55:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:55:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:18.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all mgr 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all mon 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all crash 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: pgmap v142: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all rgw 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:55:18.349 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.350 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.351 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:55:18.352 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:55:18.353 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:18.354 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.457 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.457 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:55:18.460 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:18.460 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all mgr 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all mon 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all crash 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: pgmap v142: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all rgw 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:55:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:55:18.464 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:55:18.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:18.467 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all mgr 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all mon 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all crash 2024-09-14T03:55:18.468 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: pgmap v142: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all rgw 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:55:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:18.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:19.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T03:55:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all nfs 2024-09-14T03:55:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:55:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:55:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:55:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all grafana 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all loki 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all promtail 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Finalizing container_image settings 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: Upgrade: Complete! 2024-09-14T03:55:19.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[112165]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:19.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all nfs 2024-09-14T03:55:19.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all grafana 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all loki 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all promtail 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Finalizing container_image settings 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: Upgrade: Complete! 2024-09-14T03:55:19.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:19 smithi001 ceph-mon[107399]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:19.593 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all nfs 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all grafana 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all loki 2024-09-14T03:55:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all promtail 2024-09-14T03:55:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Finalizing container_image settings 2024-09-14T03:55:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: Upgrade: Complete! 2024-09-14T03:55:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:19 smithi012 ceph-mon[101180]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[112165]: pgmap v143: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-09-14T03:55:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[107399]: pgmap v143: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-09-14T03:55:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:20 smithi012 ceph-mon[101180]: pgmap v143: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-09-14T03:55:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:55:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:22 smithi001 ceph-mon[112165]: pgmap v144: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-14T03:55:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:22 smithi001 ceph-mon[107399]: pgmap v144: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-14T03:55:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:22 smithi012 ceph-mon[101180]: pgmap v144: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-14T03:55:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:24 smithi001 ceph-mon[112165]: pgmap v145: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:24 smithi001 ceph-mon[107399]: pgmap v145: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:24 smithi012 ceph-mon[101180]: pgmap v145: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:26 smithi001 ceph-mon[112165]: pgmap v146: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:26 smithi001 ceph-mon[107399]: pgmap v146: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:26 smithi012 ceph-mon[101180]: pgmap v146: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-14T03:55:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:26] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-09-14T03:55:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:28 smithi001 ceph-mon[112165]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T03:55:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:28 smithi001 ceph-mon[107399]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T03:55:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:28 smithi012 ceph-mon[101180]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T03:55:30.398 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:30 smithi012 ceph-mon[101180]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-09-14T03:55:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:30 smithi001 ceph-mon[107399]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-09-14T03:55:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:30 smithi001 ceph-mon[112165]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-09-14T03:55:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:32 smithi001 ceph-mon[107399]: pgmap v149: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:32 smithi001 ceph-mon[112165]: pgmap v149: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:32 smithi012 ceph-mon[101180]: pgmap v149: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T03:55:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:34 smithi001 ceph-mon[107399]: pgmap v150: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:34 smithi001 ceph-mon[112165]: pgmap v150: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:34 smithi012 ceph-mon[101180]: pgmap v150: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:36 smithi001 ceph-mon[107399]: pgmap v151: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:36 smithi001 ceph-mon[112165]: pgmap v151: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:36 smithi012 ceph-mon[101180]: pgmap v151: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T03:55:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:36] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.51.0" 2024-09-14T03:55:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:38 smithi001 ceph-mon[107399]: pgmap v152: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:38 smithi001 ceph-mon[112165]: pgmap v152: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:38 smithi012 ceph-mon[101180]: pgmap v152: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:40 smithi001 ceph-mon[112165]: pgmap v153: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:40 smithi001 ceph-mon[107399]: pgmap v153: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:40 smithi012 ceph-mon[101180]: pgmap v153: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:42 smithi001 ceph-mon[112165]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:42 smithi001 ceph-mon[107399]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:42 smithi012 ceph-mon[101180]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:44 smithi001 ceph-mon[107399]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:44.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:44 smithi001 ceph-mon[112165]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:44 smithi012 ceph-mon[101180]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:46 smithi001 ceph-mon[107399]: pgmap v156: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:46 smithi001 ceph-mon[112165]: pgmap v156: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:46 smithi012 ceph-mon[101180]: pgmap v156: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T03:55:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:46] "GET /metrics HTTP/1.1" 200 36200 "" "Prometheus/2.51.0" 2024-09-14T03:55:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:48 smithi001 ceph-mon[107399]: pgmap v157: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:55:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:48 smithi001 ceph-mon[112165]: pgmap v157: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:55:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:48 smithi012 ceph-mon[101180]: pgmap v157: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T03:55:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:50 smithi001 ceph-mon[107399]: pgmap v158: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-14T03:55:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:50 smithi001 ceph-mon[112165]: pgmap v158: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-14T03:55:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:50 smithi012 ceph-mon[101180]: pgmap v158: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-14T03:55:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:51 smithi001 ceph-mon[107399]: pgmap v159: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:51 smithi001 ceph-mon[112165]: pgmap v159: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:51 smithi012 ceph-mon[101180]: pgmap v159: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:55:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:54 smithi001 ceph-mon[107399]: pgmap v160: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:54.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:54 smithi001 ceph-mon[112165]: pgmap v160: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:54 smithi012 ceph-mon[101180]: pgmap v160: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:56 smithi001 ceph-mon[107399]: pgmap v161: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:56 smithi001 ceph-mon[112165]: pgmap v161: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:56 smithi012 ceph-mon[101180]: pgmap v161: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T03:55:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:55:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:55:56] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.51.0" 2024-09-14T03:55:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:58 smithi001 ceph-mon[112165]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:58 smithi001 ceph-mon[107399]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:55:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:58 smithi012 ceph-mon[101180]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:56:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:55:59 smithi012 ceph-mon[101180]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:56:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:55:59 smithi001 ceph-mon[107399]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:56:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:55:59 smithi001 ceph-mon[112165]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-09-14T03:56:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:02 smithi001 ceph-mon[107399]: pgmap v164: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:02 smithi001 ceph-mon[112165]: pgmap v164: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:02 smithi012 ceph-mon[101180]: pgmap v164: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:04 smithi001 ceph-mon[112165]: pgmap v165: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:04 smithi001 ceph-mon[107399]: pgmap v165: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:04 smithi012 ceph-mon[101180]: pgmap v165: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:05.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:05.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:06.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:06 smithi001 ceph-mon[107399]: pgmap v166: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:06.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:06 smithi001 ceph-mon[112165]: pgmap v166: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:06.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:06 smithi012 ceph-mon[101180]: pgmap v166: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:06] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.51.0" 2024-09-14T03:56:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:08 smithi001 ceph-mon[107399]: pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:56:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:08 smithi001 ceph-mon[112165]: pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:56:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:08 smithi012 ceph-mon[101180]: pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-14T03:56:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:09 smithi001 ceph-mon[107399]: pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:56:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:09 smithi001 ceph-mon[112165]: pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:56:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:09 smithi012 ceph-mon[101180]: pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-14T03:56:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:12 smithi001 ceph-mon[107399]: pgmap v169: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:56:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:12 smithi001 ceph-mon[112165]: pgmap v169: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:56:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:12 smithi012 ceph-mon[101180]: pgmap v169: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T03:56:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:14 smithi001 ceph-mon[107399]: pgmap v170: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:14 smithi001 ceph-mon[112165]: pgmap v170: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:14 smithi012 ceph-mon[101180]: pgmap v170: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:16 smithi001 ceph-mon[107399]: pgmap v171: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:16 smithi001 ceph-mon[112165]: pgmap v171: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:16 smithi012 ceph-mon[101180]: pgmap v171: 89 pgs: 89 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-14T03:56:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:16] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.51.0" 2024-09-14T03:56:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:17 smithi001 ceph-mon[107399]: pgmap v172: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:56:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:17 smithi001 ceph-mon[112165]: pgmap v172: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:56:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:17 smithi012 ceph-mon[101180]: pgmap v172: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T03:56:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:18.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:19.474 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:19 smithi012 ceph-mon[101180]: pgmap v173: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-14T03:56:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:19 smithi001 ceph-mon[107399]: pgmap v173: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-14T03:56:19.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:19 smithi001 ceph-mon[112165]: pgmap v173: 89 pgs: 89 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-14T03:56:20.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:20.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:20.489 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:21 smithi012 ceph-mon[101180]: pgmap v174: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:21.707 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:21 smithi001 ceph-mon[107399]: pgmap v174: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:21.707 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:21 smithi001 ceph-mon[112165]: pgmap v174: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T03:56:24.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[112165]: pgmap v175: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[107399]: pgmap v175: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:24.456 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:24 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:24 smithi012 ceph-mon[101180]: pgmap v175: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:24 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:24 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:25.713 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[107399]: pgmap v176: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[112165]: pgmap v176: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:26 smithi012 ceph-mon[101180]: pgmap v176: 89 pgs: 89 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-14T03:56:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:26] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.51.0" 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:27.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:27.811 INFO:teuthology.orchestra.run.smithi001.stdout:wait for half of osds to upgrade 2024-09-14T03:56:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:28 smithi001 ceph-mon[107399]: pgmap v177: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-14T03:56:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:28 smithi001 ceph-mon[112165]: pgmap v177: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-14T03:56:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:28 smithi012 ceph-mon[101180]: pgmap v177: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-09-14T03:56:28.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-09-14T03:56:29.266 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:30 smithi001 ceph-mon[107399]: pgmap v178: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-14T03:56:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:30 smithi001 ceph-mon[112165]: pgmap v178: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-14T03:56:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:30 smithi012 ceph-mon[101180]: pgmap v178: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-14T03:56:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:32 smithi012 ceph-mon[101180]: pgmap v179: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2024-09-14T03:56:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:32 smithi001 ceph-mon[112165]: pgmap v179: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2024-09-14T03:56:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:32 smithi001 ceph-mon[107399]: pgmap v179: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2024-09-14T03:56:33.215 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:33 smithi001 ceph-mon[112165]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:33.215 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:33 smithi001 ceph-mon[107399]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:33 smithi012 ceph-mon[101180]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:34 smithi001 ceph-mon[112165]: pgmap v180: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:34 smithi001 ceph-mon[107399]: pgmap v180: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:34 smithi012 ceph-mon[101180]: pgmap v180: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:34.679 INFO:teuthology.orchestra.run.smithi001.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:35.601 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5626d550-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:35.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[107399]: pgmap v181: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:35.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:35 smithi001 ceph-mon[112165]: pgmap v181: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:36.008 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:56:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:36.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:35 smithi012 ceph-mon[101180]: pgmap v181: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T03:56:36.809 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:36] "GET /metrics HTTP/1.1" 200 36191 "" "Prometheus/2.51.0" 2024-09-14T03:56:38.324 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:38 smithi001 ceph-mon[107399]: pgmap v182: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T03:56:38.324 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:38 smithi001 ceph-mon[112165]: pgmap v182: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T03:56:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:38 smithi012 ceph-mon[101180]: pgmap v182: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T03:56:38.890 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:56:40.050 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (7m) 14s ago 15m 18.9M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (6m) 16s ago 14m 85.7M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (13m) 14s ago 13m 23.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (13m) 16s ago 13m 16.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (8m) 16s ago 18m 456M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (9m) 14s ago 20m 518M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (6m) 14s ago 20m 55.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (3m) 16s ago 19m 38.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (5m) 14s ago 19m 42.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (8m) 14s ago 14m 10.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (8m) 16s ago 14m 11.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:56:40.051 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (18m) 14s ago 18m 632M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f f508566731a9 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (18m) 14s ago 18m 471M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (17m) 14s ago 17m 572M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (17m) 14s ago 17m 645M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (17m) 16s ago 17m 516M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (16m) 16s ago 16m 506M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (16m) 16s ago 16m 454M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (15m) 16s ago 15m 478M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:56:40.052 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (7m) 16s ago 15m 47.6M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: pgmap v183: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: pgmap v183: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.402 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.403 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.403 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.403 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:56:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: pgmap v183: 89 pgs: 89 active+clean; 819 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:56:40.751 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:56:40.751 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:56:40.751 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 10, 2024-09-14T03:56:40.752 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:56:40.753 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:56:40.753 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:56:41.338 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:56:41.338 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "0/8 daemons upgraded", 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:56:41.339 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all mon 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:56:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: Upgrade: osd.0 is safe to restart 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/785742143' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all mon 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: Upgrade: osd.0 is safe to restart 2024-09-14T03:56:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/785742143' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all mon 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:56:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: Upgrade: osd.0 is safe to restart 2024-09-14T03:56:41.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:41.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/785742143' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:42.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: pgmap v184: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2024-09-14T03:56:42.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:42.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: Upgrade: Updating osd.0 2024-09-14T03:56:42.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[107399]: Deploying daemon osd.0 on smithi001 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: pgmap v184: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: Upgrade: Updating osd.0 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:42.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:42 smithi001 ceph-mon[112165]: Deploying daemon osd.0 on smithi001 2024-09-14T03:56:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: pgmap v184: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2024-09-14T03:56:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: Upgrade: Updating osd.0 2024-09-14T03:56:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:56:42.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:42.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:42 smithi012 ceph-mon[101180]: Deploying daemon osd.0 on smithi001 2024-09-14T03:56:43.455 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:43 smithi001 ceph-mon[107399]: pgmap v185: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:43.455 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:43 smithi001 ceph-mon[112165]: pgmap v185: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:43 smithi012 ceph-mon[101180]: pgmap v185: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-mon[107399]: pgmap v186: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:46.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-mon[112165]: pgmap v186: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:46.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:46 smithi001 systemd[1]: Stopping Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:46 smithi012 ceph-mon[101180]: pgmap v186: 89 pgs: 89 active+clean; 591 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T03:56:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:46] "GET /metrics HTTP/1.1" 200 36191 "" "Prometheus/2.51.0" 2024-09-14T03:56:46.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[45360]: 2024-09-14T03:56:46.588+0000 7f24dc5d6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:46.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[45360]: 2024-09-14T03:56:46.588+0000 7f24dc5d6640 -1 osd.0 91 *** Got signal Terminated *** 2024-09-14T03:56:46.972 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[45360]: 2024-09-14T03:56:46.588+0000 7f24dc5d6640 -1 osd.0 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:47 smithi001 ceph-mon[107399]: osd.0 marked itself down and dead 2024-09-14T03:56:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:47 smithi001 ceph-mon[112165]: osd.0 marked itself down and dead 2024-09-14T03:56:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:47 smithi012 ceph-mon[101180]: osd.0 marked itself down and dead 2024-09-14T03:56:48.021 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:47 smithi001 podman[123194]: 2024-09-14 03:56:47.768567599 +0000 UTC m=+1.350670063 container died f508566731a979184b3d7b46d908530b94d11f605eff047c8fcfadae28d028d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b) 2024-09-14T03:56:48.312 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[112165]: pgmap v187: 89 pgs: 89 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T03:56:48.313 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:48.313 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[112165]: osdmap e92: 8 total, 7 up, 8 in 2024-09-14T03:56:48.313 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[107399]: pgmap v187: 89 pgs: 89 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T03:56:48.313 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:48.313 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:48 smithi001 ceph-mon[107399]: osdmap e92: 8 total, 7 up, 8 in 2024-09-14T03:56:48.313 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:48 smithi001 podman[123194]: 2024-09-14 03:56:48.110084215 +0000 UTC m=+1.692186684 container cleanup f508566731a979184b3d7b46d908530b94d11f605eff047c8fcfadae28d028d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T03:56:48.313 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:48 smithi001 bash[123194]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0 2024-09-14T03:56:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:48 smithi012 ceph-mon[101180]: pgmap v187: 89 pgs: 89 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T03:56:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:48 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:48 smithi012 ceph-mon[101180]: osdmap e92: 8 total, 7 up, 8 in 2024-09-14T03:56:48.721 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:48 smithi001 podman[123212]: 2024-09-14 03:56:48.311735227 +0000 UTC m=+0.541546317 container remove f508566731a979184b3d7b46d908530b94d11f605eff047c8fcfadae28d028d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b) 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: osdmap e93: 8 total, 7 up, 8 in 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: osdmap e93: 8 total, 7 up, 8 in 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T03:56:49.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.349 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]} : dispatch 2024-09-14T03:56:49.349 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:49 smithi001 podman[123299]: 2024-09-14 03:56:48.923047681 +0000 UTC m=+0.041067446 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:49.349 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:49 smithi001 podman[123299]: 2024-09-14 03:56:49.063330993 +0000 UTC m=+0.181350756 container create e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231) 2024-09-14T03:56:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: osdmap e93: 8 total, 7 up, 8 in 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]} : dispatch 2024-09-14T03:56:49.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]} : dispatch 2024-09-14T03:56:50.039 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:49 smithi001 podman[123299]: 2024-09-14 03:56:49.733842828 +0000 UTC m=+0.851862592 container init e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:50.039 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:49 smithi001 podman[123299]: 2024-09-14 03:56:49.743194656 +0000 UTC m=+0.861214414 container start e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T03:56:50.039 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:49 smithi001 podman[123299]: 2024-09-14 03:56:49.833669803 +0000 UTC m=+0.951689570 container attach e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: pgmap v190: 89 pgs: 15 stale+active+clean, 74 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 469 op/s 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-14T03:56:50.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: osdmap e94: 8 total, 7 up, 8 in 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: pgmap v190: 89 pgs: 15 stale+active+clean, 74 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 469 op/s 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2024-09-14T03:56:50.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]}]': finished 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: osdmap e94: 8 total, 7 up, 8 in 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:50.474 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:50 smithi001 conmon[123311]: conmon e14d2948ac41b0f6d202 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea.scope/memory.events 2024-09-14T03:56:50.474 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:50 smithi001 podman[123299]: 2024-09-14 03:56:50.040882898 +0000 UTC m=+1.158902656 container died e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T03:56:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: pgmap v190: 89 pgs: 15 stale+active+clean, 74 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 469 op/s 2024-09-14T03:56:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.5", "id": [3, 0]}]': finished 2024-09-14T03:56:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [7, 0]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 0]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 0]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: osdmap e94: 8 total, 7 up, 8 in 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:51 smithi012 ceph-mon[101180]: osdmap e95: 8 total, 7 up, 8 in 2024-09-14T03:56:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:51 smithi012 ceph-mon[101180]: pgmap v193: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 14 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 52 active+clean; 192 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 1047/9255 objects degraded (11.313%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:51 smithi012 ceph-mon[101180]: osdmap e96: 8 total, 7 up, 8 in 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[112165]: osdmap e95: 8 total, 7 up, 8 in 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[112165]: pgmap v193: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 14 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 52 active+clean; 192 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 1047/9255 objects degraded (11.313%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[112165]: osdmap e96: 8 total, 7 up, 8 in 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[107399]: osdmap e95: 8 total, 7 up, 8 in 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[107399]: pgmap v193: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 14 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 52 active+clean; 192 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 1047/9255 objects degraded (11.313%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:51.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:51 smithi001 ceph-mon[107399]: osdmap e96: 8 total, 7 up, 8 in 2024-09-14T03:56:51.973 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:51 smithi001 podman[123339]: 2024-09-14 03:56:51.648485221 +0000 UTC m=+1.599899091 container remove e14d2948ac41b0f6d202ceb8b01250b7cc7bf7046f606d4d1edd14505cc02dea (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:51.973 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:51 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-14T03:56:51.973 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:51 smithi001 systemd[1]: Stopped Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:51.973 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:51 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service: Consumed 21min 35.947s CPU time. 2024-09-14T03:56:52.328 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:52.328 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 1047/9255 objects degraded (11.313%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:52.328 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:52.328 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 1047/9255 objects degraded (11.313%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:52.328 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 systemd[1]: Starting Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:52 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:52 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 1047/9255 objects degraded (11.313%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:52.646 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 podman[123447]: 2024-09-14 03:56:52.235889785 +0000 UTC m=+0.036184250 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:52.646 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 podman[123447]: 2024-09-14 03:56:52.366744067 +0000 UTC m=+0.167038507 container create 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-14T03:56:52.646 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 podman[123447]: 2024-09-14 03:56:52.644858353 +0000 UTC m=+0.445152811 container init 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:52.774 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 125722 objects 2024-09-14T03:56:52.774 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :153.616 2024-09-14T03:56:52.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 podman[123447]: 2024-09-14 03:56:52.653378893 +0000 UTC m=+0.453673334 container start 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:56:52.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 podman[123447]: 2024-09-14 03:56:52.769804455 +0000 UTC m=+0.570098894 container attach 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T03:56:52.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:52.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 bash[123447]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:52.972 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:52.972 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:52 smithi001 bash[123447]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:53.142 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-14T03:56:53.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-14T03:56:53.364 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-mon[107399]: pgmap v195: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-mon[112165]: pgmap v195: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:53.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:56:53.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:56:53.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:56:53.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:56:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:53 smithi012 ceph-mon[101180]: pgmap v195: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:53.935 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[123460]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 bash[123447]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:56:53.936 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:53 smithi001 podman[123698]: 2024-09-14 03:56:53.664013544 +0000 UTC m=+0.021802364 container died 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:56:54.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:54 smithi001 podman[123698]: 2024-09-14 03:56:54.218080811 +0000 UTC m=+0.575869673 container remove 2ac80112e728af8d01b08634a9528ad87fd51671e8fd93b316d5bbf547f5c81f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-14T03:56:55.208 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:54 smithi001 podman[123789]: 2024-09-14 03:56:54.770285249 +0000 UTC m=+0.019316972 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:55.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 podman[123789]: 2024-09-14 03:56:55.324699576 +0000 UTC m=+0.573731289 container create 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T03:56:56.057 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 podman[123789]: 2024-09-14 03:56:55.77189411 +0000 UTC m=+1.020925822 container init 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2) 2024-09-14T03:56:56.057 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 podman[123789]: 2024-09-14 03:56:55.778993089 +0000 UTC m=+1.028024804 container start 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-14T03:56:56.057 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 ceph-osd[123875]: -- 172.21.15.1:0/1808462622 <== mon.0 v2:172.21.15.1:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5626f18efa40 con 0x5626f26be000 2024-09-14T03:56:56.058 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 bash[123789]: 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc 2024-09-14T03:56:56.058 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:55 smithi001 systemd[1]: Started Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[112165]: pgmap v196: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[107399]: pgmap v196: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:56 smithi012 ceph-mon[101180]: pgmap v196: 89 pgs: 6 remapped+peering, 2 unknown, 1 peering, 17 active+undersized, 15 active+undersized+degraded, 48 active+clean; 133 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1146/6426 objects degraded (17.834%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:56.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:56:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:56:56] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.51.0" 2024-09-14T03:56:57.268 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_2' removed 2024-09-14T03:56:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:57 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:57 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:57 smithi012 ceph-mon[101180]: pgmap v197: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 48 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[112165]: pgmap v197: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 48 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:56:57.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T03:56:57.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-mon[107399]: pgmap v197: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 48 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%); 0 B/s, 0 objects/s recovering 2024-09-14T03:56:57.722 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:56:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T03:56:57.432+0000 7f81000b1640 -1 Falling back to public interface 2024-09-14T03:56:58.037 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T03:56:58.038 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T03:56:58.039 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-14T03:56:58.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-14T03:56:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T03:56:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[107399]: osdmap e97: 8 total, 7 up, 8 in 2024-09-14T03:56:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T03:56:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:58 smithi001 ceph-mon[112165]: osdmap e97: 8 total, 7 up, 8 in 2024-09-14T03:56:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:58 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:58 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T03:56:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1502327369' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T03:56:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:58 smithi012 ceph-mon[101180]: osdmap e97: 8 total, 7 up, 8 in 2024-09-14T03:56:59.037 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:56:59.432 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:56:59 smithi001 ceph-mon[107399]: pgmap v199: 73 pgs: 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:56:59.432 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:56:59 smithi001 ceph-mon[112165]: pgmap v199: 73 pgs: 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:56:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:56:59 smithi012 ceph-mon[101180]: pgmap v199: 73 pgs: 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:01.059 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_3' created 2024-09-14T03:57:01.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3893527835' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:01 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3893527835' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:01 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.449 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3893527835' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.449 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:01 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T03:57:01.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-14T03:57:02.136 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T03:57:02.136 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[107399]: osdmap e98: 8 total, 7 up, 8 in 2024-09-14T03:57:02.136 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[107399]: pgmap v201: 89 pgs: 16 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:02.137 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[107399]: osdmap e99: 8 total, 7 up, 8 in 2024-09-14T03:57:02.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T03:57:02.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[112165]: osdmap e98: 8 total, 7 up, 8 in 2024-09-14T03:57:02.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[112165]: pgmap v201: 89 pgs: 16 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:02.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:02 smithi001 ceph-mon[112165]: osdmap e99: 8 total, 7 up, 8 in 2024-09-14T03:57:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:02 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T03:57:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:02 smithi012 ceph-mon[101180]: osdmap e98: 8 total, 7 up, 8 in 2024-09-14T03:57:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:02 smithi012 ceph-mon[101180]: pgmap v201: 89 pgs: 16 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:02 smithi012 ceph-mon[101180]: osdmap e99: 8 total, 7 up, 8 in 2024-09-14T03:57:02.647 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:57:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:03 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:57:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:03 smithi001 ceph-mon[107399]: osdmap e100: 8 total, 7 up, 8 in 2024-09-14T03:57:03.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:03 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:57:03.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:03 smithi001 ceph-mon[112165]: osdmap e100: 8 total, 7 up, 8 in 2024-09-14T03:57:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:03 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:57:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:03 smithi012 ceph-mon[101180]: osdmap e100: 8 total, 7 up, 8 in 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[107399]: pgmap v204: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[112165]: pgmap v204: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:04.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:04 smithi012 ceph-mon[101180]: pgmap v204: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:05.721 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T03:57:05.519+0000 7f81000b1640 -1 osd.0 91 log_to_monitors true 2024-09-14T03:57:05.958 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-14T03:57:06.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T03:57:05.981+0000 7f80f3e53640 -1 osd.0 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: pgmap v205: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: pgmap v205: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:05 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: pgmap v205: 89 pgs: 4 creating+peering, 12 unknown, 9 active+clean+remapped, 17 active+undersized, 7 active+undersized+degraded, 40 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:05 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:57:06.967 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:06] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.51.0" 2024-09-14T03:57:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:57:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:57:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[107399]: osdmap e101: 8 total, 7 up, 8 in 2024-09-14T03:57:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[107399]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:57:07.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:57:07.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[112165]: osdmap e101: 8 total, 7 up, 8 in 2024-09-14T03:57:07.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[112165]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:06 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4091373846' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:57:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:06 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:57:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:06 smithi012 ceph-mon[101180]: osdmap e101: 8 total, 7 up, 8 in 2024-09-14T03:57:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:06 smithi012 ceph-mon[101180]: from='osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:06 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:07.657 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-14T03:57:07.660 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T03:57:07.660 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T03:57:07.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T03:57:07.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T03:57:07.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_125658 2024-09-14T03:57:07.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:57:07.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T03:57:08.283 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[107399]: osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157] boot 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[107399]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[107399]: pgmap v208: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[112165]: osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157] boot 2024-09-14T03:57:08.284 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[112165]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:57:08.285 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:57:08.285 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:07 smithi001 ceph-mon[112165]: pgmap v208: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:07 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:07 smithi012 ceph-mon[101180]: osd.0 [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157] boot 2024-09-14T03:57:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:07 smithi012 ceph-mon[101180]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:57:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:07 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:57:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:07 smithi012 ceph-mon[101180]: pgmap v208: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:08.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 15 2749 2734 170.864 170.875 0.00583416 0.00583215 2024-09-14T03:57:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:08 smithi012 ceph-mon[101180]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:57:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:08 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:57:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:08 smithi012 ceph-mon[101180]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[112165]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[112165]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[107399]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:57:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:08 smithi001 ceph-mon[107399]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:57:09.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 6191 6175 192.954 215.062 0.00881146 0.00517304 2024-09-14T03:57:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:09 smithi001 ceph-mon[107399]: pgmap v211: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:09 smithi001 ceph-mon[112165]: pgmap v211: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:09 smithi012 ceph-mon[101180]: pgmap v211: 89 pgs: 9 active+clean+remapped, 23 active+undersized, 7 active+undersized+degraded, 50 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12/78 objects degraded (15.385%); 4/78 objects misplaced (5.128%) 2024-09-14T03:57:10.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 9656 9640 200.818 216.562 0.00312116 0.00497242 2024-09-14T03:57:11.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:57:11.035+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-14T03:57:11.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 12791 12775 199.593 195.938 0.00737818 0.00500131 2024-09-14T03:57:11.986 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:57:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[112165]: pgmap v212: 89 pgs: 2 active+undersized, 2 active+undersized+degraded, 85 active+clean; 453 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 2/21780 objects degraded (0.009%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-14T03:57:12.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:11 smithi001 ceph-mon[107399]: pgmap v212: 89 pgs: 2 active+undersized, 2 active+undersized+degraded, 85 active+clean; 453 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 2/21780 objects degraded (0.009%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-14T03:57:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:11 smithi012 ceph-mon[101180]: pgmap v212: 89 pgs: 2 active+undersized, 2 active+undersized+degraded, 85 active+clean; 453 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 2/21780 objects degraded (0.009%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:12.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 15647 15631 195.371 178.5 0.00707335 0.00511212 2024-09-14T03:57:13.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:12 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:13.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:12 smithi001 ceph-mon[107399]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:12 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:12 smithi001 ceph-mon[112165]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (7m) 9s ago 15m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (7m) 49s ago 15m 85.7M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (14m) 9s ago 14m 23.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (14m) 49s ago 14m 16.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (9m) 49s ago 19m 456M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:57:13.223 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (10m) 9s ago 21m 519M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (6m) 9s ago 21m 57.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (4m) 49s ago 19m 38.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (6m) 9s ago 19m 43.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (9m) 9s ago 15m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (9m) 49s ago 15m 11.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (18s) 9s ago 19m 11.3M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (18m) 9s ago 18m 477M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f eb54e998581e 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (18m) 9s ago 18m 573M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (18m) 9s ago 18m 651M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (17m) 49s ago 17m 516M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (17m) 49s ago 17m 506M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (16m) 49s ago 16m 454M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:57:13.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (16m) 49s ago 16m 478M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:57:13.225 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (8m) 49s ago 16m 47.6M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:57:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:12 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:12 smithi012 ceph-mon[101180]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:13.717 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 18389 18373 191.369 171.375 0.008082 0.00521811 2024-09-14T03:57:13.932 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:57:13.932 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:57:13.932 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:57:13.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 7, 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:57:13.933 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 9, 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:57:13.934 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:57:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[112165]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[112165]: pgmap v213: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[112165]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[107399]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[107399]: pgmap v213: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:14.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:13 smithi001 ceph-mon[107399]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:13 smithi012 ceph-mon[101180]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:13 smithi012 ceph-mon[101180]: pgmap v213: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:13 smithi012 ceph-mon[101180]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:14.532 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "1/8 daemons upgraded", 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:57:14.533 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:57:14.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 21043 21027 187.725 165.875 0.00558585 0.00532157 2024-09-14T03:57:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded) 2024-09-14T03:57:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T03:57:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1026647990' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded) 2024-09-14T03:57:15.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T03:57:15.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1026647990' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:14 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/21780 objects degraded (0.009%), 2 pgs degraded) 2024-09-14T03:57:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:14 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T03:57:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1026647990' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:15.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 23857 23841 186.242 175.875 0.00413451 0.00536444 2024-09-14T03:57:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:15 smithi012 ceph-mon[101180]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:15 smithi012 ceph-mon[101180]: pgmap v214: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[107399]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[107399]: pgmap v214: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[112165]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:15 smithi001 ceph-mon[112165]: pgmap v214: 89 pgs: 89 active+clean; 585 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:16.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 26461 26445 183.63 162.75 0.00524248 0.00544083 2024-09-14T03:57:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:16] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.51.0" 2024-09-14T03:57:17.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 28888 28872 180.435 151.688 0.00534583 0.00553731 2024-09-14T03:57:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:18 smithi001 ceph-mon[107399]: pgmap v215: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 160 MiB/s wr, 2.57k op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:18 smithi001 ceph-mon[112165]: pgmap v215: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 160 MiB/s wr, 2.57k op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:18 smithi012 ceph-mon[101180]: pgmap v215: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 160 MiB/s wr, 2.57k op/s; 3 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T03:57:18.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 31316 31300 177.826 151.75 0.00451207 0.00561808 2024-09-14T03:57:19.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 33703 33687 175.438 149.188 0.00817404 0.00569516 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[107399]: pgmap v216: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[112165]: pgmap v216: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:20 smithi012 ceph-mon[101180]: pgmap v216: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:20.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 35985 35969 172.913 142.625 0.00628218 0.00577764 2024-09-14T03:57:21.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 15 38091 38076 169.968 131.688 0.0189529 0.00587812 2024-09-14T03:57:22.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:22 smithi001 ceph-mon[107399]: pgmap v217: 89 pgs: 89 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 175 MiB/s wr, 2.79k op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:22.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:22 smithi001 ceph-mon[112165]: pgmap v217: 89 pgs: 89 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 175 MiB/s wr, 2.79k op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:22 smithi012 ceph-mon[101180]: pgmap v217: 89 pgs: 89 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 175 MiB/s wr, 2.79k op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T03:57:22.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 40033 40017 166.723 121.312 0.00422751 0.00599195 2024-09-14T03:57:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:23 smithi012 ceph-mon[101180]: pgmap v218: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:57:23.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 15 41970 41955 163.872 121.125 0.0261883 0.00609596 2024-09-14T03:57:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:23 smithi001 ceph-mon[107399]: pgmap v218: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:57:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:23 smithi001 ceph-mon[112165]: pgmap v218: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-14T03:57:24.718 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 44048 44032 161.868 129.812 0.00732832 0.00617265 2024-09-14T03:57:25.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 46035 46019 159.774 124.188 0.00634705 0.00625322 2024-09-14T03:57:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:26 smithi001 ceph-mon[107399]: pgmap v219: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-14T03:57:26.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:26.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:26 smithi001 ceph-mon[112165]: pgmap v219: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-14T03:57:26.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:26 smithi012 ceph-mon[101180]: pgmap v219: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-09-14T03:57:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:57:26.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 47891 47875 157.47 116 0.00906132 0.00634482 2024-09-14T03:57:26.954 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:26] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:57:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:27 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:27 smithi001 ceph-mon[107399]: Upgrade: osd.1 is safe to restart 2024-09-14T03:57:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:27 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:27 smithi001 ceph-mon[112165]: Upgrade: osd.1 is safe to restart 2024-09-14T03:57:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:27 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:57:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:27 smithi012 ceph-mon[101180]: Upgrade: osd.1 is safe to restart 2024-09-14T03:57:27.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:57:27.718678+0000 min lat: 0.00122907 max lat: 0.279577 avg lat: 0.00643717 2024-09-14T03:57:27.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:57:27.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 49695 49679 155.233 112.75 0.00411448 0.00643717 2024-09-14T03:57:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:28 smithi001 ceph-mon[107399]: pgmap v220: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 190 MiB/s wr, 3.05k op/s 2024-09-14T03:57:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:28 smithi001 ceph-mon[112165]: pgmap v220: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 190 MiB/s wr, 3.05k op/s 2024-09-14T03:57:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:28 smithi012 ceph-mon[101180]: pgmap v220: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 190 MiB/s wr, 3.05k op/s 2024-09-14T03:57:28.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 51450 51434 153.064 109.688 0.0102912 0.00652756 2024-09-14T03:57:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: Upgrade: Updating osd.1 2024-09-14T03:57:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:57:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: Deploying daemon osd.1 on smithi001 2024-09-14T03:57:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:29 smithi012 ceph-mon[101180]: pgmap v221: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:57:29.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 53215 53199 151.12 110.312 0.0161905 0.00661116 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: Upgrade: Updating osd.1 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: Deploying daemon osd.1 on smithi001 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[112165]: pgmap v221: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: Upgrade: Updating osd.1 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:57:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: Deploying daemon osd.1 on smithi001 2024-09-14T03:57:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:29 smithi001 ceph-mon[107399]: pgmap v221: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T03:57:30.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 54940 54924 149.236 107.812 0.00635876 0.00669562 2024-09-14T03:57:31.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 56612 56596 147.372 104.5 0.0106608 0.00677998 2024-09-14T03:57:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:32 smithi001 ceph-mon[107399]: pgmap v222: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-14T03:57:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:32 smithi001 ceph-mon[112165]: pgmap v222: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-14T03:57:32.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:32 smithi001 systemd[1]: Stopping Ceph osd.1 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:32 smithi012 ceph-mon[101180]: pgmap v222: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-14T03:57:32.719 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 58120 58104 145.247 94.25 0.0120011 0.00687943 2024-09-14T03:57:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:33 smithi001 ceph-mon[107399]: osd.1 marked itself down and dead 2024-09-14T03:57:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:33 smithi001 ceph-mon[112165]: osd.1 marked itself down and dead 2024-09-14T03:57:33.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[51716]: 2024-09-14T03:57:33.008+0000 7fed9b32b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:33.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[51716]: 2024-09-14T03:57:33.008+0000 7fed9b32b640 -1 osd.1 104 *** Got signal Terminated *** 2024-09-14T03:57:33.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:33 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[51716]: 2024-09-14T03:57:33.008+0000 7fed9b32b640 -1 osd.1 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:57:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:33 smithi012 ceph-mon[101180]: osd.1 marked itself down and dead 2024-09-14T03:57:33.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 58909 58893 141.557 49.3125 0.00265888 0.00692556 2024-09-14T03:57:34.097 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:33 smithi001 podman[128911]: 2024-09-14 03:57:33.827237008 +0000 UTC m=+1.189730380 container died eb54e998581e6e257258a7281e398b31366d1afd591a014d7d11765e943c9180 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b) 2024-09-14T03:57:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[112165]: pgmap v223: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T03:57:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:57:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[112165]: osdmap e105: 8 total, 7 up, 8 in 2024-09-14T03:57:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[107399]: pgmap v223: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T03:57:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:57:34.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:34 smithi001 ceph-mon[107399]: osdmap e105: 8 total, 7 up, 8 in 2024-09-14T03:57:34.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:34 smithi001 podman[128911]: 2024-09-14 03:57:34.253482666 +0000 UTC m=+1.615976037 container cleanup eb54e998581e6e257258a7281e398b31366d1afd591a014d7d11765e943c9180 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:34.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:34 smithi001 bash[128911]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1 2024-09-14T03:57:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:34 smithi012 ceph-mon[101180]: pgmap v223: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T03:57:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:34 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:57:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:34 smithi012 ceph-mon[101180]: osdmap e105: 8 total, 7 up, 8 in 2024-09-14T03:57:34.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 59803 59787 138.381 55.875 0.00705857 0.00722143 2024-09-14T03:57:34.971 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:34 smithi001 podman[128930]: 2024-09-14 03:57:34.508944953 +0000 UTC m=+0.682208926 container remove eb54e998581e6e257258a7281e398b31366d1afd591a014d7d11765e943c9180 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[112165]: osdmap e106: 8 total, 7 up, 8 in 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[112165]: pgmap v226: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[107399]: osdmap e106: 8 total, 7 up, 8 in 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:35.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:35 smithi001 ceph-mon[107399]: pgmap v226: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:57:35.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.098643852 +0000 UTC m=+0.039387933 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:35.472 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.25698321 +0000 UTC m=+0.197727280 container create 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T03:57:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:35 smithi012 ceph-mon[101180]: osdmap e106: 8 total, 7 up, 8 in 2024-09-14T03:57:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:35.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:35 smithi012 ceph-mon[101180]: pgmap v226: 89 pgs: 13 stale+active+clean, 76 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T03:57:35.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 61688 61672 137.646 117.812 0.00599786 0.00725997 2024-09-14T03:57:35.842 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.518894277 +0000 UTC m=+0.459638350 container init 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:57:35.842 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.527027505 +0000 UTC m=+0.467771575 container start 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:57:35.842 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.618672655 +0000 UTC m=+0.559416728 container attach 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.build-date=20240731) 2024-09-14T03:57:36.178 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:35 smithi001 podman[129014]: 2024-09-14 03:57:35.840394352 +0000 UTC m=+0.781138418 container died 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-14T03:57:36.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 63517 63501 136.841 114.312 0.026858 0.00730137 2024-09-14T03:57:36.945 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:36] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.51.0" 2024-09-14T03:57:36.945 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:36 smithi001 podman[129057]: 2024-09-14 03:57:36.608993063 +0000 UTC m=+0.757218420 container remove 275d134cf16c602bbe1b9a902f292869e85e7469b59b798479d9307d4b2a69a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:57:36.945 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:36 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-14T03:57:36.945 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:36 smithi001 systemd[1]: Stopped Ceph osd.1 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:36.945 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:36 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.1.service: Consumed 17min 31.863s CPU time. 2024-09-14T03:57:37.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:36 smithi001 systemd[1]: Starting Ceph osd.1 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:37.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-mon[112165]: pgmap v227: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:37.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 27052/185049 objects degraded (14.619%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:37.540 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-mon[107399]: pgmap v227: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:37.540 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 27052/185049 objects degraded (14.619%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:37.541 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 podman[129151]: 2024-09-14 03:57:37.17590086 +0000 UTC m=+0.033386471 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:37.541 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 podman[129151]: 2024-09-14 03:57:37.285283675 +0000 UTC m=+0.142769267 container create 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:57:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:37 smithi012 ceph-mon[101180]: pgmap v227: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:37 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 27052/185049 objects degraded (14.619%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:37.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 65294 65278 135.982 111.062 0.00921106 0.00734884 2024-09-14T03:57:37.971 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 podman[129151]: 2024-09-14 03:57:37.538982463 +0000 UTC m=+0.396468069 container init 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-14T03:57:37.971 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 podman[129151]: 2024-09-14 03:57:37.546283301 +0000 UTC m=+0.403768892 container start 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-14T03:57:37.971 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 podman[129151]: 2024-09-14 03:57:37.6554696 +0000 UTC m=+0.512955197 container attach 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:57:37.971 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:57:37.972 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 bash[129151]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:57:37.972 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:57:37.972 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:37 smithi001 bash[129151]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:57:38.418 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:57:38.418 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:57:38.418 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:57:38.418 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:57:38.418 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:57:38.419 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:57:38.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 66971 66955 134.976 104.812 0.017718 0.00740252 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate[129164]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 bash[129151]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:57:38.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:38 smithi001 podman[129151]: 2024-09-14 03:57:38.473235481 +0000 UTC m=+1.330721073 container died 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:57:39.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 68245 68229 133.246 79.625 0.00711927 0.00748642 2024-09-14T03:57:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:40 smithi001 ceph-mon[107399]: pgmap v228: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:40 smithi001 ceph-mon[112165]: pgmap v228: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:40 smithi012 ceph-mon[101180]: pgmap v228: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27052/185049 objects degraded (14.619%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:40.720 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 15 69723 69708 132.009 92.4375 0.0136069 0.00757023 2024-09-14T03:57:41.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 71322 71306 131.064 99.875 0.00388215 0.00762488 2024-09-14T03:57:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:42 smithi001 ceph-mon[107399]: pgmap v229: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 28600/203439 objects degraded (14.058%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:42 smithi001 ceph-mon[112165]: pgmap v229: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 28600/203439 objects degraded (14.058%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:42 smithi012 ceph-mon[101180]: pgmap v229: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 28600/203439 objects degraded (14.058%); 16 KiB/s, 0 objects/s recovering 2024-09-14T03:57:42.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 72941 72925 130.21 101.188 0.0076618 0.00767498 2024-09-14T03:57:43.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:43 smithi001 ceph-mon[107399]: pgmap v230: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 30545/209274 objects degraded (14.596%); 13 KiB/s, 0 objects/s recovering 2024-09-14T03:57:43.343 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:43 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 30545/209274 objects degraded (14.596%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:43.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:43 smithi001 ceph-mon[112165]: pgmap v230: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 30545/209274 objects degraded (14.596%); 13 KiB/s, 0 objects/s recovering 2024-09-14T03:57:43.343 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:43 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 30545/209274 objects degraded (14.596%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:43.344 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:43 smithi001 podman[129309]: 2024-09-14 03:57:43.030104014 +0000 UTC m=+4.548495819 container remove 39343412b4b48e49b91055cd2ab9e174966bfa0485dcfc1332b56e3e47baf935 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:43 smithi012 ceph-mon[101180]: pgmap v230: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 30545/209274 objects degraded (14.596%); 13 KiB/s, 0 objects/s recovering 2024-09-14T03:57:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:43 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 30545/209274 objects degraded (14.596%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:43.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:43 smithi001 podman[129367]: 2024-09-14 03:57:43.246049065 +0000 UTC m=+0.041720383 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:43.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:43 smithi001 podman[129367]: 2024-09-14 03:57:43.406513574 +0000 UTC m=+0.202184860 container create dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-14T03:57:43.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 74543 74527 129.374 100.125 0.00612449 0.00772424 2024-09-14T03:57:44.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:43 smithi001 podman[129367]: 2024-09-14 03:57:43.976418551 +0000 UTC m=+0.772089846 container init dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:57:44.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:43 smithi001 podman[129367]: 2024-09-14 03:57:43.986013906 +0000 UTC m=+0.781685194 container start dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T03:57:44.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:44 smithi001 bash[129367]: dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 2024-09-14T03:57:44.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:44 smithi001 systemd[1]: Started Ceph osd.1 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:44.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 15 75999 75984 128.338 91.0625 0.0106392 0.00778597 2024-09-14T03:57:45.094 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:57:45.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.375 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.588 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:45.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:45.721 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T03:57:45.372+0000 7f8f98b5b640 -1 Falling back to public interface 2024-09-14T03:57:45.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 77390 77374 127.247 86.875 0.0151929 0.0078536 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (8m) 42s ago 16m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (7m) 82s ago 16m 85.7M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (15m) 42s ago 15m 23.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (15m) 82s ago 15m 16.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (10m) 82s ago 20m 456M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (10m) 42s ago 21m 519M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (7m) 42s ago 21m 57.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (4m) 82s ago 20m 38.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (6m) 42s ago 20m 43.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:57:46.134 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (9m) 42s ago 15m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (9m) 82s ago 15m 11.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (51s) 42s ago 19m 11.3M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 starting - - - 2481M 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (19m) 42s ago 19m 573M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (18m) 42s ago 18m 651M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (18m) 82s ago 18m 516M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (17m) 82s ago 17m 506M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (17m) 82s ago 17m 454M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (17m) 82s ago 17m 478M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:57:46.135 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (8m) 82s ago 16m 47.6M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:57:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:46 smithi001 ceph-mon[107399]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:46 smithi001 ceph-mon[107399]: pgmap v231: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 30545/209274 objects degraded (14.596%); 12 KiB/s, 0 objects/s recovering 2024-09-14T03:57:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:46 smithi001 ceph-mon[112165]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:46 smithi001 ceph-mon[112165]: pgmap v231: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 30545/209274 objects degraded (14.596%); 12 KiB/s, 0 objects/s recovering 2024-09-14T03:57:46.508 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:46 smithi012 ceph-mon[101180]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:46.508 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:46 smithi012 ceph-mon[101180]: pgmap v231: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 30545/209274 objects degraded (14.596%); 12 KiB/s, 0 objects/s recovering 2024-09-14T03:57:46.709 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:57:46.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:57:46.711 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:57:46.711 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T03:57:46.711 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:57:46.711 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:57:46.711 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:57:46.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 78661 78645 126.021 79.4375 0.00341141 0.00792937 2024-09-14T03:57:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:46] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.51.0" 2024-09-14T03:57:47.191 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "1/8 daemons upgraded", 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:57:47.192 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:57:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[107399]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[107399]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1972170878' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[112165]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[112165]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1972170878' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:47.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:47 smithi012 ceph-mon[101180]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:47 smithi012 ceph-mon[101180]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:47.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1972170878' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:47.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:57:47.720990+0000 min lat: 0.00122907 max lat: 1.04797 avg lat: 0.00799909 2024-09-14T03:57:47.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:57:47.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 79977 79961 124.926 82.25 0.00710993 0.00799909 2024-09-14T03:57:48.460 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:48 smithi001 ceph-mon[107399]: pgmap v232: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 33900/232275 objects degraded (14.595%); 11 KiB/s, 0 objects/s recovering 2024-09-14T03:57:48.460 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:48 smithi001 ceph-mon[107399]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:48.460 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:48 smithi001 ceph-mon[112165]: pgmap v232: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 33900/232275 objects degraded (14.595%); 11 KiB/s, 0 objects/s recovering 2024-09-14T03:57:48.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:48 smithi001 ceph-mon[112165]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:48.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:48 smithi012 ceph-mon[101180]: pgmap v232: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 33900/232275 objects degraded (14.595%); 11 KiB/s, 0 objects/s recovering 2024-09-14T03:57:48.494 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:48 smithi012 ceph-mon[101180]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:48.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 81356 81340 123.981 86.1875 0.00759888 0.00806061 2024-09-14T03:57:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 33900/232275 objects degraded (14.595%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 33900/232275 objects degraded (14.595%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:49.535 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 33900/232275 objects degraded (14.595%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:49.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 82734 82718 123.08 86.125 0.00930435 0.00811964 2024-09-14T03:57:50.388 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[107399]: pgmap v233: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 33900/232275 objects degraded (14.595%) 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[112165]: pgmap v233: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 33900/232275 objects degraded (14.595%) 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:50.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:50 smithi012 ceph-mon[101180]: pgmap v233: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 33900/232275 objects degraded (14.595%) 2024-09-14T03:57:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:50.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 84039 84023 122.114 81.5625 0.00583865 0.00818319 2024-09-14T03:57:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[107399]: pgmap v234: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 35137/247104 objects degraded (14.220%) 2024-09-14T03:57:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:51 smithi001 ceph-mon[112165]: pgmap v234: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 35137/247104 objects degraded (14.220%) 2024-09-14T03:57:51.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 85151 85135 120.918 69.5 0.011048 0.00826498 2024-09-14T03:57:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:51 smithi012 ceph-mon[101180]: pgmap v234: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 35137/247104 objects degraded (14.220%) 2024-09-14T03:57:52.721 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 86357 86341 119.906 75.375 0.0149085 0.00833477 2024-09-14T03:57:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:53.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T03:57:53.450+0000 7f8f98b5b640 -1 osd.1 104 log_to_monitors true 2024-09-14T03:57:53.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 87755 87739 119.199 87.375 0.014026 0.00838393 2024-09-14T03:57:53.886 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[107399]: pgmap v235: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:53.886 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[107399]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:53.887 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[107399]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:53.887 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[112165]: pgmap v235: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:53.887 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[112165]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:53.887 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-mon[112165]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:53.887 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 03:57:53 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T03:57:53.663+0000 7f8f90905640 -1 osd.1 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:57:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:53 smithi012 ceph-mon[101180]: pgmap v235: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:53 smithi012 ceph-mon[101180]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:53 smithi012 ceph-mon[101180]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:57:54.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 89080 89064 118.424 82.8125 0.00502486 0.00842399 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[107399]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[107399]: osdmap e107: 8 total, 7 up, 8 in 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[107399]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[107399]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 36814/252135 objects degraded (14.601%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[112165]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:57:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[112165]: osdmap e107: 8 total, 7 up, 8 in 2024-09-14T03:57:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[112165]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[112165]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 36814/252135 objects degraded (14.601%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:54 smithi012 ceph-mon[101180]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:57:55.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:54 smithi012 ceph-mon[101180]: osdmap e107: 8 total, 7 up, 8 in 2024-09-14T03:57:55.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:54 smithi012 ceph-mon[101180]: from='osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:55.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:54 smithi012 ceph-mon[101180]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T03:57:55.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 36814/252135 objects degraded (14.601%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:57:55.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 89080 89064 115.957 0 - 0.00842399 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:57:55.867+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422] boot 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:57:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: pgmap v238: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[107399]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422] boot 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: pgmap v238: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:55.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:55 smithi001 ceph-mon[112165]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: osd.1 [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422] boot 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: pgmap v238: 89 pgs: 27 active+undersized, 11 active+undersized+degraded, 51 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 36814/252135 objects degraded (14.601%) 2024-09-14T03:57:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:55 smithi012 ceph-mon[101180]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:57:56.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 89080 89064 113.59 0 - 0.00842399 2024-09-14T03:57:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:57:56] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.51.0" 2024-09-14T03:57:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[112165]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:56.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:57:56.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:56 smithi001 ceph-mon[107399]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:57:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:57:57.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:56 smithi012 ceph-mon[101180]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:57:57.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 89440 89424 111.769 7.5 0.0352417 0.00894041 2024-09-14T03:57:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:57 smithi012 ceph-mon[101180]: pgmap v241: 89 pgs: 6 remapped+peering, 13 peering, 9 active+undersized, 1 active+undersized+degraded, 60 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:57:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:57 smithi001 ceph-mon[107399]: pgmap v241: 89 pgs: 6 remapped+peering, 13 peering, 9 active+undersized, 1 active+undersized+degraded, 60 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:57:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:57 smithi001 ceph-mon[112165]: pgmap v241: 89 pgs: 6 remapped+peering, 13 peering, 9 active+undersized, 1 active+undersized+degraded, 60 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:57:58.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 90451 90435 110.816 63.1875 0.0213578 0.00901767 2024-09-14T03:57:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:58 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5206/266202 objects degraded (1.956%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:57:59.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:58 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5206/266202 objects degraded (1.956%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:57:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:58 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5206/266202 objects degraded (1.956%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:57:59.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 91485 91469 109.928 64.625 0.0208466 0.00909192 2024-09-14T03:58:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:57:59 smithi001 ceph-mon[107399]: pgmap v242: 89 pgs: 6 remapped+peering, 13 peering, 1 active+undersized+degraded, 69 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:58:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:57:59 smithi001 ceph-mon[112165]: pgmap v242: 89 pgs: 6 remapped+peering, 13 peering, 1 active+undersized+degraded, 69 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:58:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:57:59 smithi012 ceph-mon[101180]: pgmap v242: 89 pgs: 6 remapped+peering, 13 peering, 1 active+undersized+degraded, 69 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 5206/266202 objects degraded (1.956%) 2024-09-14T03:58:00.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 92540 92524 109.097 65.9375 0.00482174 0.00915888 2024-09-14T03:58:01.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 93566 93550 108.265 64.125 0.0128348 0.00923085 2024-09-14T03:58:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:02 smithi001 ceph-mon[107399]: pgmap v243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 5573/274608 objects degraded (2.029%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T03:58:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:02 smithi001 ceph-mon[112165]: pgmap v243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 5573/274608 objects degraded (2.029%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T03:58:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:02 smithi012 ceph-mon[101180]: pgmap v243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 5573/274608 objects degraded (2.029%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T03:58:02.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 94555 94539 107.42 61.8125 0.0376244 0.00930225 2024-09-14T03:58:03.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 95591 95575 106.658 64.75 0.00937747 0.00937029 2024-09-14T03:58:04.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:04 smithi012 ceph-mon[101180]: pgmap v244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 13127/277863 objects degraded (4.724%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T03:58:04.335 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 13127/277863 objects degraded (4.724%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:04 smithi001 ceph-mon[107399]: pgmap v244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 13127/277863 objects degraded (4.724%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T03:58:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 13127/277863 objects degraded (4.724%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:04 smithi001 ceph-mon[112165]: pgmap v244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 13127/277863 objects degraded (4.724%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T03:58:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 13127/277863 objects degraded (4.724%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:04.722 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 96590 96574 105.882 62.4375 0.00599439 0.0094386 2024-09-14T03:58:05.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 97554 97538 105.095 60.25 0.0141644 0.00950977 2024-09-14T03:58:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:58:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[107399]: pgmap v245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 13127/277863 objects degraded (4.724%); 1007 KiB/s, 15 objects/s recovering 2024-09-14T03:58:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:58:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:05 smithi001 ceph-mon[112165]: pgmap v245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 13127/277863 objects degraded (4.724%); 1007 KiB/s, 15 objects/s recovering 2024-09-14T03:58:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:58:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:05 smithi012 ceph-mon[101180]: pgmap v245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 13127/277863 objects degraded (4.724%); 1007 KiB/s, 15 objects/s recovering 2024-09-14T03:58:06.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 98558 98542 104.377 62.75 0.0278255 0.00957495 2024-09-14T03:58:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:06] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-09-14T03:58:07.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:58:07.722895+0000 min lat: 0.00122907 max lat: 2.7753 avg lat: 0.00963847 2024-09-14T03:58:07.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:58:07.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 99573 99557 103.695 63.4375 0.0090649 0.00963847 2024-09-14T03:58:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:08 smithi012 ceph-mon[101180]: pgmap v246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12870/292806 objects degraded (4.395%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T03:58:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:08 smithi001 ceph-mon[107399]: pgmap v246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12870/292806 objects degraded (4.395%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T03:58:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:08 smithi001 ceph-mon[112165]: pgmap v246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 12870/292806 objects degraded (4.395%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T03:58:08.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 100549 100533 102.995 61 0.0268417 0.0097032 2024-09-14T03:58:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12870/292806 objects degraded (4.395%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12870/292806 objects degraded (4.395%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12870/292806 objects degraded (4.395%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:09.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 101543 101527 102.336 62.125 0.0126946 0.00976672 2024-09-14T03:58:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:10 smithi012 ceph-mon[101180]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12870/292806 objects degraded (4.395%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T03:58:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:10 smithi001 ceph-mon[107399]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12870/292806 objects degraded (4.395%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T03:58:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:10 smithi001 ceph-mon[112165]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12870/292806 objects degraded (4.395%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T03:58:10.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 102501 102485 101.662 59.875 0.00555955 0.00983025 2024-09-14T03:58:11.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:10 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:58:10.884+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:11 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:11 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:11 smithi012 ceph-mon[101180]: pgmap v248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12617/303957 objects degraded (4.151%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[107399]: pgmap v248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12617/303957 objects degraded (4.151%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:11.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:11 smithi001 ceph-mon[112165]: pgmap v248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12617/303957 objects degraded (4.151%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:58:11.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 103446 103430 100.996 59.0625 0.0300519 0.00989506 2024-09-14T03:58:12.723 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 104395 104379 100.354 59.3125 0.0285316 0.00995927 2024-09-14T03:58:13.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 105350 105334 99.7382 59.6875 0.021801 0.01002 2024-09-14T03:58:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:14 smithi012 ceph-mon[101180]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12617/307662 objects degraded (4.101%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:14 smithi001 ceph-mon[107399]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12617/307662 objects degraded (4.101%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:14 smithi001 ceph-mon[112165]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12617/307662 objects degraded (4.101%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:14.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 106296 106280 99.132 59.125 0.00917576 0.0100823 2024-09-14T03:58:15.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 107237 107221 98.539 58.8125 0.00404339 0.0101419 2024-09-14T03:58:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:16 smithi012 ceph-mon[101180]: pgmap v250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:16.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:16 smithi001 ceph-mon[112165]: pgmap v250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:16 smithi001 ceph-mon[107399]: pgmap v250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 12617/307662 objects degraded (4.101%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:58:16.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 108105 108089 97.897 54.25 0.00546092 0.0102091 2024-09-14T03:58:16.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:16] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-09-14T03:58:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:17 smithi012 ceph-mon[101180]: pgmap v251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 12340/321906 objects degraded (3.833%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T03:58:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:17 smithi001 ceph-mon[107399]: pgmap v251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 12340/321906 objects degraded (3.833%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T03:58:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:17 smithi001 ceph-mon[112165]: pgmap v251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 12340/321906 objects degraded (3.833%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T03:58:17.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 15 108970 108955 97.2717 54.125 0.0140737 0.0102746 2024-09-14T03:58:17.811 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:58:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:18 smithi012 ceph-mon[101180]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:18 smithi001 ceph-mon[107399]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:18 smithi001 ceph-mon[112165]: from='client.54226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:18.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 109765 109749 96.6005 49.625 0.0204589 0.0103455 2024-09-14T03:58:18.937 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:58:18.937 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (8m) 28s ago 16m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:58:18.937 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (8m) 28s ago 16m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (15m) 28s ago 15m 23.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (15m) 28s ago 15m 16.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (10m) 28s ago 20m 457M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (11m) 28s ago 22m 520M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (7m) 28s ago 22m 58.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (5m) 28s ago 21m 39.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (7m) 28s ago 20m 46.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (10m) 28s ago 16m 11.5M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (10m) 28s ago 16m 10.7M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (84s) 28s ago 20m 248M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (35s) 28s ago 19m 12.3M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (19m) 28s ago 19m 589M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (19m) 28s ago 19m 644M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:58:18.938 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (18m) 28s ago 18m 547M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:58:18.939 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (18m) 28s ago 18m 608M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:58:18.939 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (18m) 28s ago 18m 488M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:58:18.939 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (17m) 28s ago 17m 476M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:58:18.939 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (9m) 28s ago 17m 47.7M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:58:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:19 smithi012 ceph-mon[101180]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12340/321906 objects degraded (3.833%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:19 smithi012 ceph-mon[101180]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:19 smithi012 ceph-mon[101180]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 12340/321906 objects degraded (3.833%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:58:19.626 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:58:19.627 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[107399]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12340/321906 objects degraded (3.833%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[107399]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[107399]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 12340/321906 objects degraded (3.833%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[112165]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12340/321906 objects degraded (3.833%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:19.628 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[112165]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:19.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:19 smithi001 ceph-mon[112165]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 12340/321906 objects degraded (3.833%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T03:58:19.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 110571 110555 95.9584 50.375 0.022591 0.0104151 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:58:20.197 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:58:20.198 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:58:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2710646716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[112165]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2710646716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:20.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:20 smithi001 ceph-mon[107399]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2710646716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:20 smithi012 ceph-mon[101180]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:20.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 111407 111391 95.3596 52.25 0.0301407 0.0104802 2024-09-14T03:58:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:21 smithi012 ceph-mon[101180]: pgmap v253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 12052/331350 objects degraded (3.637%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:58:21.652 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:21 smithi001 ceph-mon[112165]: pgmap v253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 12052/331350 objects degraded (3.637%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:58:21.652 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:21 smithi001 ceph-mon[107399]: pgmap v253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 12052/331350 objects degraded (3.637%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:58:21.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 112225 112209 94.7618 51.125 0.00530205 0.0105463 2024-09-14T03:58:22.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 113005 112989 94.1482 48.75 0.0099752 0.0106154 2024-09-14T03:58:23.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 113884 113868 93.6322 54.9375 0.020754 0.0106746 2024-09-14T03:58:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:24 smithi012 ceph-mon[101180]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12052/334434 objects degraded (3.604%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:24 smithi001 ceph-mon[112165]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12052/334434 objects degraded (3.604%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:24 smithi001 ceph-mon[107399]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12052/334434 objects degraded (3.604%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:24.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 114780 114764 93.1434 56 0.0102425 0.0107299 2024-09-14T03:58:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:25 smithi012 ceph-mon[101180]: pgmap v255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:25 smithi001 ceph-mon[112165]: pgmap v255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:25 smithi001 ceph-mon[107399]: pgmap v255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 12052/334434 objects degraded (3.604%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T03:58:25.724 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 115648 115632 92.6448 54.25 0.0132907 0.0107881 2024-09-14T03:58:26.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:25 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:58:25.901+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:26 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:26 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:26.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:26.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 116551 116535 92.1864 56.4375 0.00662567 0.0108401 2024-09-14T03:58:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:26] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-14T03:58:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:27 smithi012 ceph-mon[101180]: pgmap v256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 11766/347382 objects degraded (3.387%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:27 smithi001 ceph-mon[112165]: pgmap v256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 11766/347382 objects degraded (3.387%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:27 smithi001 ceph-mon[107399]: pgmap v256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 11766/347382 objects degraded (3.387%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:27.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T03:58:27.724748+0000 min lat: 0.00122907 max lat: 2.7753 avg lat: 0.0108985 2024-09-14T03:58:27.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T03:58:27.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 117407 117391 91.7027 53.5 0.0190826 0.0108985 2024-09-14T03:58:28.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 118291 118275 91.2526 55.25 0.00881619 0.0109533 2024-09-14T03:58:29.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:28 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11766/347382 objects degraded (3.387%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:29.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:28 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11766/347382 objects degraded (3.387%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:29.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:28 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11766/347382 objects degraded (3.387%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:29.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 119155 119139 90.7983 54 0.00551429 0.0110078 2024-09-14T03:58:30.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:29 smithi001 ceph-mon[112165]: pgmap v257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 11766/347382 objects degraded (3.387%); 3.0 MiB/s, 47 objects/s recovering 2024-09-14T03:58:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:29 smithi001 ceph-mon[107399]: pgmap v257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 11766/347382 objects degraded (3.387%); 3.0 MiB/s, 47 objects/s recovering 2024-09-14T03:58:30.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:29 smithi012 ceph-mon[101180]: pgmap v257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 11766/347382 objects degraded (3.387%); 3.0 MiB/s, 47 objects/s recovering 2024-09-14T03:58:30.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 120037 120021 90.3684 55.125 0.00979416 0.0110599 2024-09-14T03:58:31.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 120913 120897 89.9443 54.75 0.043827 0.0111116 2024-09-14T03:58:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:32 smithi001 ceph-mon[107399]: pgmap v258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 11458/357069 objects degraded (3.209%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:58:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:32 smithi001 ceph-mon[112165]: pgmap v258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 11458/357069 objects degraded (3.209%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:58:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:32 smithi012 ceph-mon[101180]: pgmap v258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 11458/357069 objects degraded (3.209%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:58:32.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 121513 121497 89.3273 37.5 0.0045351 0.0111445 2024-09-14T03:58:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:33 smithi001 ceph-mon[107399]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:58:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:33 smithi001 ceph-mon[107399]: pgmap v260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 11458/360450 objects degraded (3.179%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:58:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:33 smithi001 ceph-mon[112165]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:58:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:33 smithi001 ceph-mon[112165]: pgmap v260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 11458/360450 objects degraded (3.179%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:58:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:33 smithi012 ceph-mon[101180]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:58:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:33 smithi012 ceph-mon[101180]: pgmap v260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 11458/360450 objects degraded (3.179%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:58:33.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 121902 121886 88.5713 24.3125 0.0118097 0.0112845 2024-09-14T03:58:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:34 smithi012 ceph-mon[101180]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:58:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11458/360450 objects degraded (3.179%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:34 smithi001 ceph-mon[112165]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:58:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11458/360450 objects degraded (3.179%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:34 smithi001 ceph-mon[107399]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:58:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11458/360450 objects degraded (3.179%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:34.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 122742 122726 88.1566 52.5 0.00683697 0.0113379 2024-09-14T03:58:35.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:35.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:35 smithi012 ceph-mon[101180]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s; 11373/360867 objects degraded (3.152%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T03:58:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:35 smithi001 ceph-mon[107399]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s; 11373/360867 objects degraded (3.152%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T03:58:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:35 smithi001 ceph-mon[112165]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 561 op/s; 11373/360867 objects degraded (3.152%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T03:58:35.725 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 123567 123551 87.7407 51.5625 0.0149047 0.0113915 2024-09-14T03:58:36.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 124398 124382 87.3384 51.9375 0.0130052 0.0114445 2024-09-14T03:58:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:36] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.51.0" 2024-09-14T03:58:37.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 90 5 125236 125231 86.9575 53.0625 0.0190275 0.0114945 2024-09-14T03:58:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:38 smithi012 ceph-mon[101180]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 11052/370743 objects degraded (2.981%); 5.6 MiB/s, 89 objects/s recovering 2024-09-14T03:58:38.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:38 smithi001 ceph-mon[107399]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 11052/370743 objects degraded (2.981%); 5.6 MiB/s, 89 objects/s recovering 2024-09-14T03:58:38.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:38 smithi001 ceph-mon[112165]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 11052/370743 objects degraded (2.981%); 5.6 MiB/s, 89 objects/s recovering 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0118 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 125236 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 86.9581 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 45.2433 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 216.562 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1391 2024-09-14T03:58:38.726 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 723.894 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3465 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0114948 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.035674 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 2.7753 2024-09-14T03:58:38.727 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00122907 2024-09-14T03:58:38.732 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T03:58:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11052/370743 objects degraded (2.981%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:39.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11052/370743 objects degraded (2.981%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:39.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11052/370743 objects degraded (2.981%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:40 smithi012 ceph-mon[101180]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11052/371406 objects degraded (2.976%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T03:58:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:40 smithi001 ceph-mon[112165]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11052/371406 objects degraded (2.976%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T03:58:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:40 smithi001 ceph-mon[107399]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11052/371406 objects degraded (2.976%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T03:58:41.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:40 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:58:40.918+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:41 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:41 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:41 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:41 smithi012 ceph-mon[101180]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 114 B/s rd, 34 MiB/s wr, 664 op/s; 10622/371733 objects degraded (2.857%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T03:58:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[107399]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 114 B/s rd, 34 MiB/s wr, 664 op/s; 10622/371733 objects degraded (2.857%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T03:58:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:58:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:58:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:58:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:41 smithi001 ceph-mon[112165]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 114 B/s rd, 34 MiB/s wr, 664 op/s; 10622/371733 objects degraded (2.857%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T03:58:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:43 smithi012 ceph-mon[101180]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 678 op/s; 10622/370776 objects degraded (2.865%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:43 smithi001 ceph-mon[112165]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 678 op/s; 10622/370776 objects degraded (2.865%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:43 smithi001 ceph-mon[107399]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 678 op/s; 10622/370776 objects degraded (2.865%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10622/370776 objects degraded (2.865%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10622/370776 objects degraded (2.865%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10622/370776 objects degraded (2.865%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:45 smithi012 ceph-mon[101180]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 590 op/s; 10622/369990 objects degraded (2.871%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:45 smithi001 ceph-mon[112165]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 590 op/s; 10622/369990 objects degraded (2.871%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:45 smithi001 ceph-mon[107399]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 590 op/s; 10622/369990 objects degraded (2.871%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T03:58:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:46] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.51.0" 2024-09-14T03:58:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:47 smithi012 ceph-mon[101180]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 907 op/s; 10374/358029 objects degraded (2.898%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:47 smithi001 ceph-mon[112165]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 907 op/s; 10374/358029 objects degraded (2.898%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:47 smithi001 ceph-mon[107399]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 907 op/s; 10374/358029 objects degraded (2.898%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T03:58:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:48 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10374/358029 objects degraded (2.898%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:48 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10374/358029 objects degraded (2.898%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:48 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10374/358029 objects degraded (2.898%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:49 smithi001 ceph-mon[107399]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.8 MiB/s wr, 654 op/s; 10374/357258 objects degraded (2.904%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:58:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:49 smithi001 ceph-mon[112165]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.8 MiB/s wr, 654 op/s; 10374/357258 objects degraded (2.904%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:58:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:49 smithi012 ceph-mon[101180]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.8 MiB/s wr, 654 op/s; 10374/357258 objects degraded (2.904%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:58:50.797 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:58:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:51.983 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:58:51.983 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (9m) 61s ago 17m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:58:51.983 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (8m) 61s ago 17m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:58:51.983 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (16m) 61s ago 16m 23.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:58:51.983 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (16m) 61s ago 16m 16.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (11m) 61s ago 21m 457M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (11m) 61s ago 22m 520M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (8m) 61s ago 23m 58.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (5m) 61s ago 21m 39.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (7m) 61s ago 21m 46.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (10m) 61s ago 16m 11.5M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (10m) 61s ago 16m 10.7M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (117s) 61s ago 20m 248M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (68s) 61s ago 20m 12.3M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (20m) 61s ago 20m 589M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (19m) 61s ago 19m 644M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (19m) 61s ago 19m 547M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:58:51.984 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (18m) 61s ago 18m 608M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:58:51.985 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (18m) 61s ago 18m 488M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:58:51.985 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (18m) 61s ago 18m 476M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:58:51.985 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (10m) 61s ago 17m 47.7M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:58:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:51 smithi001 ceph-mon[112165]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:51 smithi001 ceph-mon[112165]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 785 op/s; 10138/351855 objects degraded (2.881%); 4.8 MiB/s, 76 objects/s recovering 2024-09-14T03:58:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:51 smithi001 ceph-mon[107399]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:51 smithi001 ceph-mon[107399]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 785 op/s; 10138/351855 objects degraded (2.881%); 4.8 MiB/s, 76 objects/s recovering 2024-09-14T03:58:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:51 smithi012 ceph-mon[101180]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:51 smithi012 ceph-mon[101180]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 785 op/s; 10138/351855 objects degraded (2.881%); 4.8 MiB/s, 76 objects/s recovering 2024-09-14T03:58:52.665 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:58:52.665 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:58:52.665 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:58:52.666 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T03:58:52.667 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:58:52.667 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:58:52.667 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:58:52.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[107399]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:52.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[107399]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:52.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2850679275' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[112165]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[112165]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2850679275' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:53.247 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:58:53.247 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:58:53.248 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:58:53.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:52 smithi012 ceph-mon[101180]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:53.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:52 smithi012 ceph-mon[101180]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:53.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2850679275' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:53 smithi001 ceph-mon[112165]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 1.4 MiB/s wr, 762 op/s; 10138/345855 objects degraded (2.931%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:53 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10138/351855 objects degraded (2.881%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:53 smithi001 ceph-mon[107399]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 1.4 MiB/s wr, 762 op/s; 10138/345855 objects degraded (2.931%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:53 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10138/351855 objects degraded (2.881%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:53 smithi012 ceph-mon[101180]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 1.4 MiB/s wr, 762 op/s; 10138/345855 objects degraded (2.931%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:53 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10138/351855 objects degraded (2.881%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T03:58:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:54 smithi001 ceph-mon[112165]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:54 smithi001 ceph-mon[107399]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:54 smithi012 ceph-mon[101180]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:56.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:55 smithi001 ceph-mon[112165]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 10138/345186 objects degraded (2.937%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:56.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:55 smithi001 ceph-mon[107399]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 10138/345186 objects degraded (2.937%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:55 smithi012 ceph-mon[101180]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 10138/345186 objects degraded (2.937%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T03:58:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:58:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:58:56] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.51.0" 2024-09-14T03:58:57.212 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:57 smithi001 ceph-mon[112165]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 9882/334701 objects degraded (2.952%); 3.9 MiB/s, 61 objects/s recovering 2024-09-14T03:58:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:57 smithi001 ceph-mon[107399]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 9882/334701 objects degraded (2.952%); 3.9 MiB/s, 61 objects/s recovering 2024-09-14T03:58:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:57 smithi012 ceph-mon[101180]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 9882/334701 objects degraded (2.952%); 3.9 MiB/s, 61 objects/s recovering 2024-09-14T03:58:59.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:58 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9882/334701 objects degraded (2.952%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:58:59.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:58 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9882/334701 objects degraded (2.952%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:58:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:58 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9882/334701 objects degraded (2.952%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:00.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:58:59 smithi001 ceph-mon[107399]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 9882/333954 objects degraded (2.959%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T03:59:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:58:59 smithi001 ceph-mon[112165]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 9882/333954 objects degraded (2.959%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T03:59:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:58:59 smithi012 ceph-mon[101180]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 9882/333954 objects degraded (2.959%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T03:59:01.565 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:01 smithi012 ceph-mon[101180]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 9558/328731 objects degraded (2.908%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T03:59:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:01 smithi001 ceph-mon[112165]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 9558/328731 objects degraded (2.908%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T03:59:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:01 smithi001 ceph-mon[107399]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 9558/328731 objects degraded (2.908%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T03:59:02.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:59:02.826+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:59:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:02.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:02.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:02.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:03.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:03.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T03:59:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[107399]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9558/322995 objects degraded (2.959%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:03.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:03.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:03.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:59:03.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:59:03.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:03 smithi001 ceph-mon[112165]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9558/322995 objects degraded (2.959%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T03:59:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:59:04.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:03 smithi012 ceph-mon[101180]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9558/322995 objects degraded (2.959%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T03:59:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9558/328731 objects degraded (2.908%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:04 smithi001 ceph-mon[107399]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:59:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9558/328731 objects degraded (2.908%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:04 smithi001 ceph-mon[112165]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:59:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9558/328731 objects degraded (2.908%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:04 smithi012 ceph-mon[101180]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:59:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:05 smithi001 ceph-mon[107399]: pgmap v279: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 9440/322059 objects degraded (2.931%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T03:59:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:05 smithi001 ceph-mon[112165]: pgmap v279: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 9440/322059 objects degraded (2.931%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T03:59:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:05 smithi012 ceph-mon[101180]: pgmap v279: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 9440/322059 objects degraded (2.931%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T03:59:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:06] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.51.0" 2024-09-14T03:59:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:08 smithi012 ceph-mon[101180]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 9182/313230 objects degraded (2.931%); 5.5 MiB/s, 87 objects/s recovering 2024-09-14T03:59:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:08 smithi001 ceph-mon[107399]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 9182/313230 objects degraded (2.931%); 5.5 MiB/s, 87 objects/s recovering 2024-09-14T03:59:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:08 smithi001 ceph-mon[112165]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 9182/313230 objects degraded (2.931%); 5.5 MiB/s, 87 objects/s recovering 2024-09-14T03:59:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9182/313230 objects degraded (2.931%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:09 smithi012 ceph-mon[101180]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9182/311706 objects degraded (2.946%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9182/313230 objects degraded (2.931%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:09 smithi001 ceph-mon[107399]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9182/311706 objects degraded (2.946%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9182/313230 objects degraded (2.931%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:09 smithi001 ceph-mon[112165]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 9182/311706 objects degraded (2.946%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:11 smithi012 ceph-mon[101180]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 9182/306690 objects degraded (2.994%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:11 smithi001 ceph-mon[107399]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 9182/306690 objects degraded (2.994%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:11 smithi001 ceph-mon[112165]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 9182/306690 objects degraded (2.994%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T03:59:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:13 smithi012 ceph-mon[101180]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 8969/300483 objects degraded (2.985%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T03:59:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:13 smithi001 ceph-mon[107399]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 8969/300483 objects degraded (2.985%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T03:59:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:13 smithi001 ceph-mon[112165]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 8969/300483 objects degraded (2.985%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T03:59:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8969/300483 objects degraded (2.985%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8969/300483 objects degraded (2.985%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8969/300483 objects degraded (2.985%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:15 smithi012 ceph-mon[101180]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 8969/298905 objects degraded (3.001%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:59:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:15 smithi001 ceph-mon[107399]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 8969/298905 objects degraded (3.001%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:59:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:15 smithi001 ceph-mon[112165]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 8969/298905 objects degraded (3.001%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T03:59:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:16] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.51.0" 2024-09-14T03:59:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:17 smithi012 ceph-mon[101180]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 8763/288090 objects degraded (3.042%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:17 smithi001 ceph-mon[107399]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 8763/288090 objects degraded (3.042%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:17 smithi001 ceph-mon[112165]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 8763/288090 objects degraded (3.042%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:18.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:17 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:59:17.843+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:18 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:18 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:18.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:18 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8763/288090 objects degraded (3.042%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:19 smithi012 ceph-mon[101180]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8763/286512 objects degraded (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8763/288090 objects degraded (3.042%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:19 smithi001 ceph-mon[107399]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8763/286512 objects degraded (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8763/288090 objects degraded (3.042%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:19 smithi001 ceph-mon[112165]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8763/286512 objects degraded (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:20.621 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:20.622 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:21 smithi012 ceph-mon[101180]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 8763/281772 objects degraded (3.110%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:21.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:21 smithi001 ceph-mon[107399]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 8763/281772 objects degraded (3.110%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:21.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:21 smithi001 ceph-mon[112165]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 8763/281772 objects degraded (3.110%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T03:59:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:23 smithi012 ceph-mon[101180]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 8522/275709 objects degraded (3.091%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:23 smithi001 ceph-mon[107399]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 8522/275709 objects degraded (3.091%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:23 smithi001 ceph-mon[112165]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 8522/275709 objects degraded (3.091%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:23.858 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:59:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8522/275709 objects degraded (3.091%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:24 smithi012 ceph-mon[101180]: from='client.54265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8522/275709 objects degraded (3.091%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:24 smithi001 ceph-mon[112165]: from='client.54265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8522/275709 objects degraded (3.091%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:24 smithi001 ceph-mon[107399]: from='client.54265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (10m) 23s ago 17m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (9m) 22s ago 17m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (16m) 23s ago 16m 24.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (16m) 22s ago 16m 17.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (11m) 22s ago 21m 458M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (12m) 23s ago 23m 521M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (8m) 23s ago 23m 60.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:59:24.960 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (6m) 22s ago 22m 41.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (8m) 23s ago 22m 47.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (11m) 23s ago 17m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (11m) 22s ago 17m 10.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (2m) 23s ago 21m 343M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (101s) 23s ago 21m 338M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (20m) 23s ago 20m 878M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (20m) 23s ago 20m 727M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (19m) 22s ago 19m 589M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (19m) 22s ago 19m 924M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (19m) 22s ago 19m 547M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (18m) 22s ago 18m 522M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:59:24.961 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (10m) 22s ago 18m 49.8M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:59:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:25 smithi012 ceph-mon[101180]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:25 smithi012 ceph-mon[101180]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:25 smithi012 ceph-mon[101180]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8522/274209 objects degraded (3.108%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T03:59:25.637 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[112165]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[112165]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[112165]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8522/274209 objects degraded (3.108%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T03:59:25.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[107399]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[107399]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:25.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:25 smithi001 ceph-mon[107399]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8522/274209 objects degraded (3.108%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:25.640 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T03:59:25.641 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:59:25.642 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:59:25.642 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:59:26.210 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:59:26.210 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:59:26.210 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:59:26.211 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:59:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2740773120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:26 smithi001 ceph-mon[107399]: from='client.54280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2740773120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:26 smithi001 ceph-mon[112165]: from='client.54280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2740773120' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:26 smithi012 ceph-mon[101180]: from='client.54280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:26] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-14T03:59:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:27 smithi012 ceph-mon[101180]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 8288/264504 objects degraded (3.133%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:27 smithi001 ceph-mon[107399]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 8288/264504 objects degraded (3.133%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:27 smithi001 ceph-mon[112165]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 8288/264504 objects degraded (3.133%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T03:59:29.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:28 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8288/264504 objects degraded (3.133%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:29.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:28 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8288/264504 objects degraded (3.133%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:29.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:28 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8288/264504 objects degraded (3.133%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:30.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:29 smithi001 ceph-mon[107399]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 8288/263166 objects degraded (3.149%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:30.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:29 smithi001 ceph-mon[112165]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 8288/263166 objects degraded (3.149%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:30.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:29 smithi012 ceph-mon[101180]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 8288/263166 objects degraded (3.149%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:31 smithi012 ceph-mon[101180]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8288/258420 objects degraded (3.207%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:31 smithi001 ceph-mon[107399]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8288/258420 objects degraded (3.207%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:31 smithi001 ceph-mon[112165]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8288/258420 objects degraded (3.207%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T03:59:33.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:32 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:33.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:32 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:59:32.860+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:33.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:32 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:32 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[107399]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7994/252099 objects degraded (3.171%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T03:59:34.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8288/258420 objects degraded (3.207%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:34.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:34.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:34.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[112165]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7994/252099 objects degraded (3.171%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T03:59:34.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:33 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8288/258420 objects degraded (3.207%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:33 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:33 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-14T03:59:34.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:33 smithi012 ceph-mon[101180]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7994/252099 objects degraded (3.171%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T03:59:34.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:33 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8288/258420 objects degraded (3.207%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:35.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:35.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:35 smithi001 ceph-mon[107399]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7994/250593 objects degraded (3.190%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:35 smithi001 ceph-mon[112165]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7994/250593 objects degraded (3.190%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:35 smithi012 ceph-mon[101180]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 7994/250593 objects degraded (3.190%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:36] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-14T03:59:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:37 smithi012 ceph-mon[101180]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7694/239631 objects degraded (3.211%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:59:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:37 smithi001 ceph-mon[107399]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7694/239631 objects degraded (3.211%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:59:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:37 smithi001 ceph-mon[112165]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7694/239631 objects degraded (3.211%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T03:59:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:39 smithi012 ceph-mon[101180]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:59:39.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7694/239631 objects degraded (3.211%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:39.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:39 smithi012 ceph-mon[101180]: pgmap v297: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7554/237648 objects degraded (3.179%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:59:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[107399]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:59:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7694/239631 objects degraded (3.211%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[107399]: pgmap v297: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7554/237648 objects degraded (3.179%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:59:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[112165]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:59:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7694/239631 objects degraded (3.211%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:39.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:39 smithi001 ceph-mon[112165]: pgmap v297: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7554/237648 objects degraded (3.179%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T03:59:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:40 smithi012 ceph-mon[101180]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:59:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:40 smithi001 ceph-mon[107399]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:59:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:40 smithi001 ceph-mon[112165]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:59:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:41 smithi012 ceph-mon[101180]: pgmap v299: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7554/233793 objects degraded (3.231%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:41 smithi001 ceph-mon[107399]: pgmap v299: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7554/233793 objects degraded (3.231%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:41 smithi001 ceph-mon[112165]: pgmap v299: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7554/233793 objects degraded (3.231%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T03:59:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:43 smithi012 ceph-mon[101180]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7345/229092 objects degraded (3.206%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T03:59:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:43 smithi001 ceph-mon[107399]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7345/229092 objects degraded (3.206%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T03:59:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:43 smithi001 ceph-mon[112165]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7345/229092 objects degraded (3.206%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T03:59:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7345/229092 objects degraded (3.206%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7345/229092 objects degraded (3.206%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7345/229092 objects degraded (3.206%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:45 smithi012 ceph-mon[101180]: pgmap v301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 7345/226980 objects degraded (3.236%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:45 smithi001 ceph-mon[112165]: pgmap v301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 7345/226980 objects degraded (3.236%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:45 smithi001 ceph-mon[107399]: pgmap v301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 7345/226980 objects degraded (3.236%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:46] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-14T03:59:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:47 smithi012 ceph-mon[101180]: pgmap v302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7177/216720 objects degraded (3.312%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:59:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:47 smithi001 ceph-mon[112165]: pgmap v302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7177/216720 objects degraded (3.312%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:59:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:47 smithi001 ceph-mon[107399]: pgmap v302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7177/216720 objects degraded (3.312%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T03:59:48.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:47 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T03:59:47.880+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T03:59:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:48 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:48 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:59:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:48 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T03:59:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7177/216720 objects degraded (3.312%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:49 smithi012 ceph-mon[101180]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 7177/214218 objects degraded (3.350%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T03:59:49.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7177/216720 objects degraded (3.312%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:49.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:49 smithi001 ceph-mon[107399]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 7177/214218 objects degraded (3.350%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T03:59:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7177/216720 objects degraded (3.312%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:49 smithi001 ceph-mon[112165]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 7177/214218 objects degraded (3.350%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T03:59:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:50.621 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:50.622 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:51 smithi012 ceph-mon[101180]: pgmap v304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 7177/209418 objects degraded (3.427%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T03:59:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:51 smithi001 ceph-mon[107399]: pgmap v304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 7177/209418 objects degraded (3.427%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T03:59:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:51 smithi001 ceph-mon[112165]: pgmap v304: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 7177/209418 objects degraded (3.427%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T03:59:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:53 smithi012 ceph-mon[101180]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 7003/204051 objects degraded (3.432%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T03:59:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:53 smithi001 ceph-mon[107399]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 7003/204051 objects degraded (3.432%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T03:59:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:53 smithi001 ceph-mon[112165]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 7003/204051 objects degraded (3.432%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T03:59:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7003/204051 objects degraded (3.432%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7003/204051 objects degraded (3.432%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7003/204051 objects degraded (3.432%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:59:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=cleanup t=2024-09-14T03:59:55.254398224Z level=info msg="Completed cleanup jobs" duration=3.697904ms 2024-09-14T03:59:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:59:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore.transactions t=2024-09-14T03:59:55.26200949Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:59:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:59:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugins.update.checker t=2024-09-14T03:59:55.424571142Z level=info msg="Update check succeeded" duration=65.178746ms 2024-09-14T03:59:55.597 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 03:59:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana.update.checker t=2024-09-14T03:59:55.429043037Z level=info msg="Update check succeeded" duration=63.335589ms 2024-09-14T03:59:55.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:55 smithi012 ceph-mon[101180]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7003/201834 objects degraded (3.470%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T03:59:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:55 smithi001 ceph-mon[107399]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7003/201834 objects degraded (3.470%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T03:59:55.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:55 smithi001 ceph-mon[112165]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7003/201834 objects degraded (3.470%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T03:59:56.809 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T03:59:56.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 03:59:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:03:59:56] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-14T03:59:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:57 smithi012 ceph-mon[101180]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:57 smithi012 ceph-mon[101180]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 6822/191676 objects degraded (3.559%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:57 smithi001 ceph-mon[107399]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:57 smithi001 ceph-mon[107399]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 6822/191676 objects degraded (3.559%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:57 smithi001 ceph-mon[112165]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:57 smithi001 ceph-mon[112165]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 6822/191676 objects degraded (3.559%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T03:59:57.920 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:59:57.920 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (10m) 56s ago 18m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (10m) 55s ago 18m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (17m) 56s ago 17m 24.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (17m) 55s ago 17m 17.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (12m) 55s ago 22m 458M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (12m) 56s ago 24m 521M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (9m) 56s ago 24m 60.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (7m) 55s ago 22m 41.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (8m) 56s ago 22m 47.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (11m) 56s ago 17m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (11m) 55s ago 17m 10.5M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (3m) 56s ago 21m 343M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (2m) 56s ago 21m 338M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T03:59:57.921 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (21m) 56s ago 21m 878M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (20m) 56s ago 20m 727M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (20m) 55s ago 20m 589M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (20m) 55s ago 20m 924M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (19m) 55s ago 19m 547M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (19m) 55s ago 19m 522M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T03:59:57.922 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (11m) 55s ago 18m 49.8M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T03:59:58.587 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:58 smithi001 ceph-mon[107399]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:58.588 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:58 smithi001 ceph-mon[107399]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:58.588 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:58 smithi001 ceph-mon[112165]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:58.588 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:58 smithi001 ceph-mon[112165]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:58.590 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:59:58.590 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T03:59:58.590 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:59:58.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:58.590 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T03:59:58.591 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:59:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:58 smithi012 ceph-mon[101180]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:58 smithi012 ceph-mon[101180]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T03:59:59.163 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/474533985' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6822/191676 objects degraded (3.559%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[107399]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[107399]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6822/189429 objects degraded (3.601%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/474533985' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6822/191676 objects degraded (3.559%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[112165]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 03:59:59 smithi001 ceph-mon[112165]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6822/189429 objects degraded (3.601%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T03:59:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/474533985' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6822/191676 objects degraded (3.559%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T03:59:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:59 smithi012 ceph-mon[101180]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 03:59:59 smithi012 ceph-mon[101180]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6822/189429 objects degraded (3.601%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: Health detail: HEALTH_WARN Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: [WRN] PG_DEGRADED: Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: pg 8.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,4] 2024-09-14T04:00:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:00 smithi012 ceph-mon[101180]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-09-14T04:00:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: Health detail: HEALTH_WARN Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: [WRN] PG_DEGRADED: Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-09-14T04:00:01.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-09-14T04:00:01.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: pg 8.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,4] 2024-09-14T04:00:01.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[107399]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-09-14T04:00:01.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: Health detail: HEALTH_WARN Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: [WRN] PG_DEGRADED: Degraded data redundancy: 6822/189429 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized 2024-09-14T04:00:01.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-09-14T04:00:01.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-09-14T04:00:01.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: pg 8.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,4] 2024-09-14T04:00:01.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:00 smithi001 ceph-mon[112165]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-09-14T04:00:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:01 smithi001 ceph-mon[107399]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6822/185103 objects degraded (3.686%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:00:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:01 smithi001 ceph-mon[112165]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6822/185103 objects degraded (3.686%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:00:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:01 smithi012 ceph-mon[101180]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6822/185103 objects degraded (3.686%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:00:03.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:03.224 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:03.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:03 smithi001 ceph-mon[107399]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6636/179958 objects degraded (3.688%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:00:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:03 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6822/185103 objects degraded (3.686%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:04.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:03 smithi001 ceph-mon[112165]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6636/179958 objects degraded (3.688%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:00:04.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:03 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6822/185103 objects degraded (3.686%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:03 smithi012 ceph-mon[101180]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6636/179958 objects degraded (3.688%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:00:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:03 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6822/185103 objects degraded (3.686%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:05.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:05 smithi012 ceph-mon[101180]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6636/177456 objects degraded (3.740%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:00:06.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:05 smithi001 ceph-mon[107399]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6636/177456 objects degraded (3.740%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:00:06.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:05 smithi001 ceph-mon[112165]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6636/177456 objects degraded (3.740%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:00:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:06] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-14T04:00:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:07 smithi012 ceph-mon[101180]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 6342/168105 objects degraded (3.773%); 3.4 MiB/s, 55 objects/s recovering 2024-09-14T04:00:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:07 smithi001 ceph-mon[107399]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 6342/168105 objects degraded (3.773%); 3.4 MiB/s, 55 objects/s recovering 2024-09-14T04:00:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:07 smithi001 ceph-mon[112165]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 6342/168105 objects degraded (3.773%); 3.4 MiB/s, 55 objects/s recovering 2024-09-14T04:00:09.207 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:08 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6342/168105 objects degraded (3.773%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:08 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6342/168105 objects degraded (3.773%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:08 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6342/168105 objects degraded (3.773%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[107399]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6342/166098 objects degraded (3.818%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[112165]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6342/166098 objects degraded (3.818%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:10.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:09 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:09 smithi012 ceph-mon[101180]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6342/166098 objects degraded (3.818%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:09 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:09 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:11 smithi012 ceph-mon[101180]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 6342/162252 objects degraded (3.909%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:11.602 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:11 smithi001 ceph-mon[112165]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 6342/162252 objects degraded (3.909%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:11.602 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:11 smithi001 ceph-mon[107399]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 6342/162252 objects degraded (3.909%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:13.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:13.223 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:00:12.818+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:13.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:14 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:14 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:14 smithi012 ceph-mon[101180]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6114/157500 objects degraded (3.882%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6342/162252 objects degraded (3.909%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[112165]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6114/157500 objects degraded (3.882%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6342/162252 objects degraded (3.909%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[107399]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6114/157500 objects degraded (3.882%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6342/162252 objects degraded (3.909%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:15 smithi012 ceph-mon[101180]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6114/155256 objects degraded (3.938%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:00:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:15 smithi001 ceph-mon[107399]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6114/155256 objects degraded (3.938%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:00:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:15 smithi001 ceph-mon[112165]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6114/155256 objects degraded (3.938%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:00:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:16] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-14T04:00:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:17 smithi012 ceph-mon[101180]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5871/145020 objects degraded (4.048%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:00:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:17 smithi001 ceph-mon[107399]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5871/145020 objects degraded (4.048%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:00:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:17 smithi001 ceph-mon[112165]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5871/145020 objects degraded (4.048%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:00:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:18 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5871/145020 objects degraded (4.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:19.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:18 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5871/145020 objects degraded (4.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:18 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5871/145020 objects degraded (4.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:20.225 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:19 smithi001 ceph-mon[107399]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5871/142623 objects degraded (4.116%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:20.226 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:19 smithi001 ceph-mon[112165]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5871/142623 objects degraded (4.116%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:19 smithi012 ceph-mon[101180]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5871/142623 objects degraded (4.116%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:00:21.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:21 smithi001 ceph-mon[107399]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T04:00:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:21 smithi001 ceph-mon[107399]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 5871/138093 objects degraded (4.251%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:00:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:21 smithi001 ceph-mon[112165]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T04:00:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:21 smithi001 ceph-mon[112165]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 5871/138093 objects degraded (4.251%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:00:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:21 smithi012 ceph-mon[101180]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T04:00:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:21 smithi012 ceph-mon[101180]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 5871/138093 objects degraded (4.251%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:00:23.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:22 smithi001 ceph-mon[107399]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T04:00:23.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:22 smithi001 ceph-mon[112165]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T04:00:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:22 smithi012 ceph-mon[101180]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T04:00:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:23 smithi001 ceph-mon[107399]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5528/133080 objects degraded (4.154%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:00:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:23 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5871/138093 objects degraded (4.251%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:23.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:23 smithi001 ceph-mon[112165]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5528/133080 objects degraded (4.154%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:00:23.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:23 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5871/138093 objects degraded (4.251%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:23 smithi012 ceph-mon[101180]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5528/133080 objects degraded (4.154%); 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:00:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:23 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5871/138093 objects degraded (4.251%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:25 smithi012 ceph-mon[101180]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5528/131136 objects degraded (4.215%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:00:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:25 smithi001 ceph-mon[107399]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5528/131136 objects degraded (4.215%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:00:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:25 smithi001 ceph-mon[112165]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5528/131136 objects degraded (4.215%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:00:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:26] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-09-14T04:00:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:27 smithi012 ceph-mon[101180]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 5381/121986 objects degraded (4.411%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:00:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:27 smithi001 ceph-mon[107399]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 5381/121986 objects degraded (4.411%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:00:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:27 smithi001 ceph-mon[112165]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 5381/121986 objects degraded (4.411%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:00:28.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:27 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:00:27.836+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:28 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:28 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:28 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:28 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5381/121986 objects degraded (4.411%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:29 smithi012 ceph-mon[101180]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5381/119577 objects degraded (4.500%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5381/121986 objects degraded (4.411%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:29 smithi001 ceph-mon[107399]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5381/119577 objects degraded (4.500%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5381/121986 objects degraded (4.411%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:29 smithi001 ceph-mon[112165]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 5381/119577 objects degraded (4.500%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:00:29.754 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T04:00:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:30 smithi012 ceph-mon[101180]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:30 smithi001 ceph-mon[107399]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:30 smithi001 ceph-mon[112165]: from='client.54298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (11m) 18s ago 18m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (10m) 21s ago 18m 84.9M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (17m) 18s ago 17m 24.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (17m) 21s ago 17m 17.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (12m) 21s ago 22m 458M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (13m) 18s ago 24m 522M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (9m) 18s ago 24m 62.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T04:00:30.876 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (7m) 21s ago 23m 42.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (9m) 18s ago 23m 48.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (12m) 18s ago 18m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (12m) 21s ago 18m 11.7M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (3m) 18s ago 22m 558M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (2m) 18s ago 22m 376M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (21m) 18s ago 21m 892M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (21m) 18s ago 21m 919M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (21m) 21s ago 20m 602M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (20m) 21s ago 20m 940M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (20m) 21s ago 20m 550M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (19m) 21s ago 19m 538M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T04:00:30.877 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (11m) 21s ago 19m 49.8M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:00:31.555 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:00:31.556 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:00:31.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[112165]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[112165]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.557 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[112165]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5381/114921 objects degraded (4.682%); 3.1 MiB/s, 48 objects/s recovering 2024-09-14T04:00:31.557 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[107399]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.558 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[107399]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.558 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:31 smithi001 ceph-mon[107399]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5381/114921 objects degraded (4.682%); 3.1 MiB/s, 48 objects/s recovering 2024-09-14T04:00:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:31 smithi012 ceph-mon[101180]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:31 smithi012 ceph-mon[101180]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:31 smithi012 ceph-mon[101180]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5381/114921 objects degraded (4.682%); 3.1 MiB/s, 48 objects/s recovering 2024-09-14T04:00:32.100 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:00:32.100 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:00:32.100 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T04:00:32.101 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:00:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2722089851' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:32 smithi001 ceph-mon[107399]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2722089851' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:32.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:32 smithi001 ceph-mon[112165]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2722089851' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:00:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:32 smithi012 ceph-mon[101180]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:33 smithi001 ceph-mon[107399]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5232/109596 objects degraded (4.774%); 3.4 MiB/s, 56 objects/s recovering 2024-09-14T04:00:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:33 smithi001 ceph-mon[112165]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5232/109596 objects degraded (4.774%); 3.4 MiB/s, 56 objects/s recovering 2024-09-14T04:00:33.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:33 smithi012 ceph-mon[101180]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5232/109596 objects degraded (4.774%); 3.4 MiB/s, 56 objects/s recovering 2024-09-14T04:00:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5232/109596 objects degraded (4.774%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5232/109596 objects degraded (4.774%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:34.759 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5232/109596 objects degraded (4.774%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:35.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:35.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:35 smithi012 ceph-mon[101180]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5232/107484 objects degraded (4.868%); 1.4 MiB/s, 24 objects/s recovering 2024-09-14T04:00:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:35 smithi001 ceph-mon[107399]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5232/107484 objects degraded (4.868%); 1.4 MiB/s, 24 objects/s recovering 2024-09-14T04:00:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:35 smithi001 ceph-mon[112165]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5232/107484 objects degraded (4.868%); 1.4 MiB/s, 24 objects/s recovering 2024-09-14T04:00:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:36] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-14T04:00:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:37 smithi012 ceph-mon[101180]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 5059/97356 objects degraded (5.196%); 1.4 MiB/s, 39 objects/s recovering 2024-09-14T04:00:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:37 smithi001 ceph-mon[107399]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 5059/97356 objects degraded (5.196%); 1.4 MiB/s, 39 objects/s recovering 2024-09-14T04:00:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:37 smithi001 ceph-mon[112165]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 5059/97356 objects degraded (5.196%); 1.4 MiB/s, 39 objects/s recovering 2024-09-14T04:00:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:38 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5059/97356 objects degraded (5.196%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:38 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5059/97356 objects degraded (5.196%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:38 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5059/97356 objects degraded (5.196%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:39 smithi001 ceph-mon[112165]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5059/95043 objects degraded (5.323%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:39 smithi001 ceph-mon[107399]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5059/95043 objects degraded (5.323%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:39 smithi012 ceph-mon[101180]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5059/95043 objects degraded (5.323%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:41 smithi012 ceph-mon[101180]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 5059/90306 objects degraded (5.602%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:41 smithi001 ceph-mon[107399]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 5059/90306 objects degraded (5.602%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:41 smithi001 ceph-mon[112165]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 5059/90306 objects degraded (5.602%); 688 KiB/s, 26 objects/s recovering 2024-09-14T04:00:43.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:43.222 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:00:42.852+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:43.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[107399]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 4870/84738 objects degraded (5.747%); 688 KiB/s, 42 objects/s recovering 2024-09-14T04:00:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5059/90306 objects degraded (5.602%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[112165]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 4870/84738 objects degraded (5.747%); 688 KiB/s, 42 objects/s recovering 2024-09-14T04:00:44.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:43 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5059/90306 objects degraded (5.602%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:43 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:43 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:43 smithi012 ceph-mon[101180]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 4870/84738 objects degraded (5.747%); 688 KiB/s, 42 objects/s recovering 2024-09-14T04:00:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:43 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5059/90306 objects degraded (5.602%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:45 smithi012 ceph-mon[101180]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 4870/82371 objects degraded (5.912%); 0 B/s, 30 objects/s recovering 2024-09-14T04:00:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:45 smithi001 ceph-mon[107399]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 4870/82371 objects degraded (5.912%); 0 B/s, 30 objects/s recovering 2024-09-14T04:00:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:45 smithi001 ceph-mon[112165]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 4870/82371 objects degraded (5.912%); 0 B/s, 30 objects/s recovering 2024-09-14T04:00:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:46] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-14T04:00:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:47 smithi012 ceph-mon[101180]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 988 op/s; 4686/71889 objects degraded (6.518%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:47 smithi001 ceph-mon[107399]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 988 op/s; 4686/71889 objects degraded (6.518%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:47 smithi001 ceph-mon[112165]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 988 op/s; 4686/71889 objects degraded (6.518%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:49.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:48 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4686/71889 objects degraded (6.518%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:49.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:48 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4686/71889 objects degraded (6.518%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:48 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4686/71889 objects degraded (6.518%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:49 smithi001 ceph-mon[107399]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 4686/69375 objects degraded (6.755%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:49 smithi001 ceph-mon[112165]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 4686/69375 objects degraded (6.755%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:49 smithi012 ceph-mon[101180]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 4686/69375 objects degraded (6.755%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:51 smithi012 ceph-mon[101180]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4686/64590 objects degraded (7.255%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:51 smithi001 ceph-mon[107399]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4686/64590 objects degraded (7.255%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:51 smithi001 ceph-mon[112165]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4686/64590 objects degraded (7.255%); 0 B/s, 31 objects/s recovering 2024-09-14T04:00:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:53 smithi012 ceph-mon[101180]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4511/59016 objects degraded (7.644%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:53 smithi001 ceph-mon[107399]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4511/59016 objects degraded (7.644%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:53 smithi001 ceph-mon[112165]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4511/59016 objects degraded (7.644%); 0 B/s, 45 objects/s recovering 2024-09-14T04:00:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4511/59016 objects degraded (7.644%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4511/59016 objects degraded (7.644%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4511/59016 objects degraded (7.644%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:55 smithi012 ceph-mon[101180]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4511/56598 objects degraded (7.970%); 0 B/s, 29 objects/s recovering 2024-09-14T04:00:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:55 smithi001 ceph-mon[107399]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4511/56598 objects degraded (7.970%); 0 B/s, 29 objects/s recovering 2024-09-14T04:00:55.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:55 smithi001 ceph-mon[112165]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4511/56598 objects degraded (7.970%); 0 B/s, 29 objects/s recovering 2024-09-14T04:00:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:00:56] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.51.0" 2024-09-14T04:00:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:57 smithi012 ceph-mon[101180]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 960 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 4314/46149 objects degraded (9.348%); 0 B/s, 46 objects/s recovering 2024-09-14T04:00:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:57 smithi001 ceph-mon[112165]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 960 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 4314/46149 objects degraded (9.348%); 0 B/s, 46 objects/s recovering 2024-09-14T04:00:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:57 smithi001 ceph-mon[107399]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 960 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 4314/46149 objects degraded (9.348%); 0 B/s, 46 objects/s recovering 2024-09-14T04:00:58.221 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:00:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:00:57.869+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:58 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:58 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:58 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:58.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:00:58.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:00:58.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:58 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:00:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4314/46149 objects degraded (9.348%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:00:59 smithi001 ceph-mon[112165]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 910 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4314/43728 objects degraded (9.866%); 0 B/s, 30 objects/s recovering 2024-09-14T04:00:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4314/46149 objects degraded (9.348%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:00:59 smithi001 ceph-mon[107399]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 910 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4314/43728 objects degraded (9.866%); 0 B/s, 30 objects/s recovering 2024-09-14T04:00:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4314/46149 objects degraded (9.348%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:00:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:00:59 smithi012 ceph-mon[101180]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 910 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4314/43728 objects degraded (9.866%); 0 B/s, 30 objects/s recovering 2024-09-14T04:01:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:01 smithi012 ceph-mon[101180]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 807 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4314/38775 objects degraded (11.126%); 0 B/s, 30 objects/s recovering 2024-09-14T04:01:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:01 smithi001 ceph-mon[107399]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 807 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4314/38775 objects degraded (11.126%); 0 B/s, 30 objects/s recovering 2024-09-14T04:01:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:01 smithi001 ceph-mon[112165]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 807 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4314/38775 objects degraded (11.126%); 0 B/s, 30 objects/s recovering 2024-09-14T04:01:02.698 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T04:01:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:03 smithi012 ceph-mon[101180]: from='client.54322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:03 smithi012 ceph-mon[101180]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:03 smithi012 ceph-mon[101180]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 688 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4117/33057 objects degraded (12.454%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[112165]: from='client.54322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[112165]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[112165]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 688 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4117/33057 objects degraded (12.454%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[107399]: from='client.54322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[107399]: from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:03 smithi001 ceph-mon[107399]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 688 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4117/33057 objects degraded (12.454%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (11m) 51s ago 19m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (11m) 54s ago 19m 84.9M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (18m) 51s ago 18m 24.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (18m) 54s ago 18m 17.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T04:01:03.802 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (13m) 54s ago 23m 458M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (13m) 51s ago 25m 522M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (10m) 51s ago 25m 62.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (8m) 54s ago 23m 42.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (9m) 51s ago 23m 48.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (12m) 51s ago 19m 11.2M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (12m) 54s ago 18m 11.7M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T04:01:03.803 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (4m) 51s ago 23m 558M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (3m) 51s ago 22m 376M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (22m) 51s ago 22m 892M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (21m) 51s ago 21m 919M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (21m) 54s ago 21m 602M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (21m) 54s ago 21m 940M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (20m) 54s ago 20m 550M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T04:01:03.804 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (20m) 54s ago 20m 538M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T04:01:03.805 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (12m) 54s ago 19m 49.8M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T04:01:04.465 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:01:04.466 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:01:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:04 smithi012 ceph-mon[101180]: from='client.54334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4117/33057 objects degraded (12.454%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:04 smithi001 ceph-mon[112165]: from='client.54334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4117/33057 objects degraded (12.454%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:04 smithi001 ceph-mon[107399]: from='client.54334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4117/33057 objects degraded (12.454%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:05.039 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:01:05.039 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T04:01:05.040 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:01:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/70135627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:05 smithi012 ceph-mon[101180]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:05 smithi012 ceph-mon[101180]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 642 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4117/30861 objects degraded (13.340%); 0 B/s, 32 objects/s recovering 2024-09-14T04:01:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/70135627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[107399]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[107399]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 642 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4117/30861 objects degraded (13.340%); 0 B/s, 32 objects/s recovering 2024-09-14T04:01:05.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/70135627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:05.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:05.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[112165]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:05.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:05 smithi001 ceph-mon[112165]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 642 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4117/30861 objects degraded (13.340%); 0 B/s, 32 objects/s recovering 2024-09-14T04:01:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:06] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.51.0" 2024-09-14T04:01:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:07 smithi012 ceph-mon[101180]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 439 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 3941/21144 objects degraded (18.639%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:07 smithi001 ceph-mon[112165]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 439 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 3941/21144 objects degraded (18.639%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:07 smithi001 ceph-mon[107399]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 439 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 3941/21144 objects degraded (18.639%); 0 B/s, 47 objects/s recovering 2024-09-14T04:01:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:08 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3941/21144 objects degraded (18.639%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:08 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3941/21144 objects degraded (18.639%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:08 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3941/21144 objects degraded (18.639%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:09 smithi001 ceph-mon[107399]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 388 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3941/18690 objects degraded (21.086%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:09 smithi001 ceph-mon[112165]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 388 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3941/18690 objects degraded (21.086%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:09 smithi012 ceph-mon[101180]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 388 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3941/18690 objects degraded (21.086%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:11 smithi012 ceph-mon[101180]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 287 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 3941/13821 objects degraded (28.515%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:11.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:11 smithi001 ceph-mon[107399]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 287 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 3941/13821 objects degraded (28.515%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:11 smithi001 ceph-mon[112165]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 287 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 3941/13821 objects degraded (28.515%); 0 B/s, 31 objects/s recovering 2024-09-14T04:01:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:13 smithi012 ceph-mon[101180]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T04:01:13.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:13.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:13 smithi012 ceph-mon[101180]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 167 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 3721/8055 objects degraded (46.195%); 0 B/s, 39 objects/s recovering 2024-09-14T04:01:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[107399]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T04:01:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[107399]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 167 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 3721/8055 objects degraded (46.195%); 0 B/s, 39 objects/s recovering 2024-09-14T04:01:13.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[112165]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T04:01:13.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:13.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:13 smithi001 ceph-mon[112165]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 167 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 3721/8055 objects degraded (46.195%); 0 B/s, 39 objects/s recovering 2024-09-14T04:01:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:14 smithi012 ceph-mon[101180]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T04:01:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3721/8055 objects degraded (46.195%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:14 smithi001 ceph-mon[107399]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T04:01:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3721/8055 objects degraded (46.195%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:14 smithi001 ceph-mon[112165]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T04:01:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3721/8055 objects degraded (46.195%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:14.758 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 125236 objects 2024-09-14T04:01:14.758 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :156.025 2024-09-14T04:01:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:15 smithi012 ceph-mon[101180]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 121 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3721/5841 objects degraded (63.705%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:15 smithi001 ceph-mon[107399]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 121 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3721/5841 objects degraded (63.705%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:15 smithi001 ceph-mon[112165]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 121 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3721/5841 objects degraded (63.705%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:16.966 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:16] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.51.0" 2024-09-14T04:01:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:17 smithi012 ceph-mon[101180]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 25 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3721/1260 objects degraded (295.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:17 smithi001 ceph-mon[107399]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 25 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3721/1260 objects degraded (295.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:17 smithi001 ceph-mon[112165]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 25 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3721/1260 objects degraded (295.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:01:17.821 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-14T04:01:17.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-14T04:01:18.065 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:01:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:18.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:01:18.609+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:19.801 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3721/1260 objects degraded (295.317%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[107399]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3037/78 objects degraded (3893.590%); 0 B/s, 112 objects/s recovering 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:19.802 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:19.803 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:01:19.803 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3721/1260 objects degraded (295.317%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:19.803 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:19 smithi001 ceph-mon[112165]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3037/78 objects degraded (3893.590%); 0 B/s, 112 objects/s recovering 2024-09-14T04:01:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3721/1260 objects degraded (295.317%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:01:19.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:19 smithi012 ceph-mon[101180]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3037/78 objects degraded (3893.590%); 0 B/s, 112 objects/s recovering 2024-09-14T04:01:20.528 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_3' removed 2024-09-14T04:01:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2517358876' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:20 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:20.930 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:20.930 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2517358876' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:20.930 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:20.930 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:20.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2517358876' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:20.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:20 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:01:21.453 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T04:01:21.454 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-14T04:01:21.454 DEBUG:teuthology.parallel:result is None 2024-09-14T04:01:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:01:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[107399]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T04:01:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[107399]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:01:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[112165]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T04:01:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:21 smithi001 ceph-mon[112165]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:21 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:01:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:21 smithi012 ceph-mon[101180]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T04:01:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:21 smithi012 ceph-mon[101180]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:22 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3037/78 objects degraded (3893.590%), 2 pgs degraded, 2 pgs undersized) 2024-09-14T04:01:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:22 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:01:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:22 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3037/78 objects degraded (3893.590%), 2 pgs degraded, 2 pgs undersized) 2024-09-14T04:01:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:22 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:01:22.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:22 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3037/78 objects degraded (3893.590%), 2 pgs degraded, 2 pgs undersized) 2024-09-14T04:01:22.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:22 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:01:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:23 smithi012 ceph-mon[101180]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:23 smithi001 ceph-mon[107399]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:23.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:23 smithi001 ceph-mon[112165]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:25 smithi012 ceph-mon[101180]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:25 smithi001 ceph-mon[107399]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:25 smithi001 ceph-mon[112165]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:26] "GET /metrics HTTP/1.1" 200 36273 "" "Prometheus/2.51.0" 2024-09-14T04:01:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:27 smithi012 ceph-mon[101180]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:27 smithi001 ceph-mon[107399]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:27 smithi001 ceph-mon[112165]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:29 smithi012 ceph-mon[101180]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:29 smithi001 ceph-mon[107399]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:29 smithi001 ceph-mon[112165]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:31.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:32.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:32 smithi012 ceph-mon[101180]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:32 smithi001 ceph-mon[107399]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:32 smithi001 ceph-mon[112165]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:33 smithi012 ceph-mon[101180]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:33 smithi001 ceph-mon[107399]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:33 smithi001 ceph-mon[112165]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:34 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:34.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:34 smithi012 ceph-mon[101180]: Upgrade: osd.2 is safe to restart 2024-09-14T04:01:34.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:34.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:34.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[107399]: Upgrade: osd.2 is safe to restart 2024-09-14T04:01:34.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:01:34.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:01:34.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:34 smithi001 ceph-mon[112165]: Upgrade: osd.2 is safe to restart 2024-09-14T04:01:35.398 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T04:01:36.000 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (12m) 18s ago 20m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (11m) 17s ago 19m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (18m) 18s ago 18m 25.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (18m) 17s ago 18m 17.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (13m) 17s ago 23m 459M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (14m) 18s ago 25m 522M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (11m) 18s ago 25m 63.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (8m) 17s ago 24m 43.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (10m) 18s ago 24m 49.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (13m) 18s ago 19m 11.4M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (13m) 17s ago 19m 11.1M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T04:01:36.001 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (4m) 18s ago 23m 559M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (3m) 18s ago 23m 406M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (22m) 18s ago 22m 889M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 9758c86f8913 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (22m) 18s ago 22m 1205M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (22m) 17s ago 22m 603M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (21m) 17s ago 21m 936M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (21m) 17s ago 21m 563M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (20m) 17s ago 20m 548M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T04:01:36.002 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (12m) 17s ago 20m 51.9M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: Upgrade: Updating osd.2 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: Deploying daemon osd.2 on smithi001 2024-09-14T04:01:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[107399]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: Upgrade: Updating osd.2 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: Deploying daemon osd.2 on smithi001 2024-09-14T04:01:36.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:35 smithi001 ceph-mon[112165]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: Upgrade: Updating osd.2 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:36.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: Deploying daemon osd.2 on smithi001 2024-09-14T04:01:36.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:35 smithi012 ceph-mon[101180]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:36.356 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:01:36.356 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T04:01:36.357 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T04:01:36.358 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T04:01:36.358 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:01:36.358 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:01:36.657 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:01:36.657 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:01:36.657 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T04:01:36.657 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T04:01:36.657 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T04:01:36.658 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T04:01:36.658 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:01:36.658 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T04:01:36.658 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:01:36.947 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:36] "GET /metrics HTTP/1.1" 200 33963 "" "Prometheus/2.51.0" 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[107399]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[107399]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[107399]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3967215269' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[112165]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[112165]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[112165]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:37 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3967215269' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:37 smithi012 ceph-mon[101180]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:37 smithi012 ceph-mon[101180]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:37 smithi012 ceph-mon[101180]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:37.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:37 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3967215269' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:38.431 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:38 smithi001 ceph-mon[107399]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:38.431 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:38 smithi001 ceph-mon[107399]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:38.432 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:38 smithi001 ceph-mon[112165]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:38.432 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:38 smithi001 ceph-mon[112165]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:38 smithi012 ceph-mon[101180]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:38 smithi012 ceph-mon[101180]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:39.072 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:38 smithi001 systemd[1]: Stopping Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:01:39.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:39 smithi001 ceph-mon[107399]: osd.2 marked itself down and dead 2024-09-14T04:01:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:39 smithi001 ceph-mon[112165]: osd.2 marked itself down and dead 2024-09-14T04:01:39.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[58000]: 2024-09-14T04:01:39.070+0000 7fde6a9d4640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:01:39.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[58000]: 2024-09-14T04:01:39.070+0000 7fde6a9d4640 -1 osd.2 121 *** Got signal Terminated *** 2024-09-14T04:01:39.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[58000]: 2024-09-14T04:01:39.070+0000 7fde6a9d4640 -1 osd.2 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:01:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:39 smithi012 ceph-mon[101180]: osd.2 marked itself down and dead 2024-09-14T04:01:40.087 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:39 smithi001 podman[136392]: 2024-09-14 04:01:39.797042533 +0000 UTC m=+0.824228745 container died 9758c86f8913986b805559c7568aa9d32a402a9d72b0c0c9d297ddfefe056037 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[112165]: osdmap e122: 8 total, 7 up, 8 in 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[112165]: pgmap v364: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[107399]: osdmap e122: 8 total, 7 up, 8 in 2024-09-14T04:01:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:40 smithi001 ceph-mon[107399]: pgmap v364: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:40.472 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:40 smithi001 podman[136392]: 2024-09-14 04:01:40.252504616 +0000 UTC m=+1.279690830 container cleanup 9758c86f8913986b805559c7568aa9d32a402a9d72b0c0c9d297ddfefe056037 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-14T04:01:40.472 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:40 smithi001 bash[136392]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2 2024-09-14T04:01:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:40 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:01:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:40 smithi012 ceph-mon[101180]: osdmap e122: 8 total, 7 up, 8 in 2024-09-14T04:01:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:40 smithi012 ceph-mon[101180]: pgmap v364: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:01:41.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:40 smithi001 podman[136415]: 2024-09-14 04:01:40.812362452 +0000 UTC m=+1.014567500 container remove 9758c86f8913986b805559c7568aa9d32a402a9d72b0c0c9d297ddfefe056037 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, RELEASE=reef-e4e227b, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:01:41.542 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:41 smithi001 ceph-mon[107399]: osdmap e123: 8 total, 7 up, 8 in 2024-09-14T04:01:41.542 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:41 smithi001 ceph-mon[112165]: osdmap e123: 8 total, 7 up, 8 in 2024-09-14T04:01:41.542 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.149621743 +0000 UTC m=+0.016586934 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:01:41.542 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.280032277 +0000 UTC m=+0.146997503 container create b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T04:01:41.542 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.54118837 +0000 UTC m=+0.408153560 container init b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T04:01:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:41 smithi012 ceph-mon[101180]: osdmap e123: 8 total, 7 up, 8 in 2024-09-14T04:01:41.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.54673841 +0000 UTC m=+0.413703591 container start b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T04:01:41.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.641019752 +0000 UTC m=+0.507984932 container attach b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-14T04:01:41.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:41 smithi001 podman[136508]: 2024-09-14 04:01:41.77599061 +0000 UTC m=+0.642955793 container died b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-14T04:01:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:42 smithi012 ceph-mon[101180]: pgmap v366: 73 pgs: 7 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:01:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:42 smithi001 ceph-mon[107399]: pgmap v366: 73 pgs: 7 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:01:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:42 smithi001 ceph-mon[112165]: pgmap v366: 73 pgs: 7 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:01:43.173 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:42 smithi001 podman[136551]: 2024-09-14 04:01:42.720542723 +0000 UTC m=+0.934789012 container remove b49f66c272942be27710d99652695b604e36fd1b78a060080a45e4c5b0e53f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T04:01:43.173 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:42 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-14T04:01:43.173 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:42 smithi001 systemd[1]: Stopped Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:01:43.173 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:42 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service: Consumed 27min 33.934s CPU time. 2024-09-14T04:01:43.173 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:42 smithi001 systemd[1]: Starting Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:01:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:43 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:01:43.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:43 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:01:43.472 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:43 smithi001 podman[136655]: 2024-09-14 04:01:43.074326376 +0000 UTC m=+0.016412057 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:01:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:43 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:01:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:44 smithi012 ceph-mon[101180]: pgmap v367: 73 pgs: 14 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:44 smithi001 ceph-mon[107399]: pgmap v367: 73 pgs: 14 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:44 smithi001 ceph-mon[112165]: pgmap v367: 73 pgs: 14 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:44.721 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:44 smithi001 podman[136655]: 2024-09-14 04:01:44.473024601 +0000 UTC m=+1.415110273 container create 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:01:45.096 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:44 smithi001 podman[136655]: 2024-09-14 04:01:44.942846481 +0000 UTC m=+1.884932156 container init 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:01:45.096 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:44 smithi001 podman[136655]: 2024-09-14 04:01:44.950173443 +0000 UTC m=+1.892259123 container start 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:01:45.096 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 podman[136655]: 2024-09-14 04:01:45.034031135 +0000 UTC m=+1.976116821 container attach 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T04:01:45.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:01:45.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:01:45.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:01:45.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:01:45.859 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:01:45.859 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:01:45.859 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:01:45.860 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:01:45.860 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:01:45.860 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[136670]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:01:46.190 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 bash[136655]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:01:46.191 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:45 smithi001 podman[136655]: 2024-09-14 04:01:45.888666028 +0000 UTC m=+2.830751699 container died 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-14T04:01:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:46 smithi001 ceph-mon[107399]: pgmap v368: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:46 smithi001 ceph-mon[112165]: pgmap v368: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:46 smithi012 ceph-mon[101180]: pgmap v368: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:46.882 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:46] "GET /metrics HTTP/1.1" 200 33963 "" "Prometheus/2.51.0" 2024-09-14T04:01:46.882 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:46 smithi001 podman[136830]: 2024-09-14 04:01:46.538903745 +0000 UTC m=+0.644141841 container remove 4f56bbb89fd2d38d07698d498c9e85e56b21919bd2968bdd7c6f2cd0d4dfaa33 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T04:01:46.882 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:46 smithi001 podman[136895]: 2024-09-14 04:01:46.658441161 +0000 UTC m=+0.016607692 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:01:46.882 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:46 smithi001 podman[136895]: 2024-09-14 04:01:46.764799762 +0000 UTC m=+0.122966325 container create 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T04:01:47.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:46 smithi001 podman[136895]: 2024-09-14 04:01:46.992455601 +0000 UTC m=+0.350622131 container init 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T04:01:47.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:46 smithi001 podman[136895]: 2024-09-14 04:01:46.997773378 +0000 UTC m=+0.355939901 container start 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:01:47.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:47 smithi001 ceph-osd[136933]: -- 172.21.15.1:0/624351843 <== mon.0 v2:172.21.15.1:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5637d0669a40 con 0x5637d0647c00 2024-09-14T04:01:47.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:47 smithi001 bash[136895]: 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 2024-09-14T04:01:47.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:47 smithi001 systemd[1]: Started Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:01:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[112165]: pgmap v369: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:48.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:48.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-mon[107399]: pgmap v369: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:48.472 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:48 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:01:48.389+0000 7f4ecf1d9640 -1 Falling back to public interface 2024-09-14T04:01:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:48.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:48 smithi012 ceph-mon[101180]: pgmap v369: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:01:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:01:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:01:50.445 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[112165]: pgmap v370: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:50.445 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:50.445 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:50.446 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[107399]: pgmap v370: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:50.446 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:50.446 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:50 smithi012 ceph-mon[101180]: pgmap v370: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:51 smithi012 ceph-mon[101180]: pgmap v371: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[107399]: pgmap v371: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.098 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.098 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.098 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[112165]: pgmap v371: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:52.098 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:52.098 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:53.589 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:53 smithi001 ceph-mon[107399]: pgmap v372: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:53.590 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:53 smithi001 ceph-mon[112165]: pgmap v372: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:53 smithi012 ceph-mon[101180]: pgmap v372: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:55.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:01:55.121+0000 7fc9b3f23640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-14T04:01:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-14T04:01:56.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:56 smithi012 ceph-mon[101180]: pgmap v373: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:56.426 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[112165]: pgmap v373: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:56.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-14T04:01:56.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-mon[107399]: pgmap v373: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:56.701 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:01:56.424+0000 7f4ecf1d9640 -1 osd.2 121 log_to_monitors true 2024-09-14T04:01:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:01:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:01:56] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.51.0" 2024-09-14T04:01:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:57 smithi012 ceph-mon[101180]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:01:57.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:57 smithi001 ceph-mon[107399]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:01:57.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:57 smithi001 ceph-mon[112165]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:01:57.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:01:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:01:57.061+0000 7f4ec6f83640 -1 osd.2 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:01:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:58 smithi012 ceph-mon[101180]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:01:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:58 smithi012 ceph-mon[101180]: osdmap e124: 8 total, 7 up, 8 in 2024-09-14T04:01:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:58 smithi012 ceph-mon[101180]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:01:58.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:58 smithi012 ceph-mon[101180]: pgmap v375: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[107399]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[107399]: osdmap e124: 8 total, 7 up, 8 in 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[107399]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[107399]: pgmap v375: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[112165]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[112165]: osdmap e124: 8 total, 7 up, 8 in 2024-09-14T04:01:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[112165]: from='osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:01:58.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:58 smithi001 ceph-mon[112165]: pgmap v375: 73 pgs: 18 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:01:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:59 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:01:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:59 smithi012 ceph-mon[101180]: osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272] boot 2024-09-14T04:01:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:59 smithi012 ceph-mon[101180]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T04:01:59.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:01:59.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:01:59 smithi012 ceph-mon[101180]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T04:01:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:01:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[107399]: osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272] boot 2024-09-14T04:01:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[107399]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T04:01:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[107399]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[112165]: osd.2 [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272] boot 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[112165]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:01:59.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:01:59 smithi001 ceph-mon[112165]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T04:02:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:00 smithi012 ceph-mon[101180]: pgmap v378: 73 pgs: 15 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:02:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:00 smithi001 ceph-mon[107399]: pgmap v378: 73 pgs: 15 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:02:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:00 smithi001 ceph-mon[112165]: pgmap v378: 73 pgs: 15 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-14T04:02:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:01 smithi012 ceph-mon[101180]: pgmap v379: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:01 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:01.620 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:01 smithi001 ceph-mon[107399]: pgmap v379: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:01.620 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:01 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:01.620 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:01 smithi001 ceph-mon[112165]: pgmap v379: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:01.620 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:01 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:03 smithi012 ceph-mon[101180]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:03 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded) 2024-09-14T04:02:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:03 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:02:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[107399]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded) 2024-09-14T04:02:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:02:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[112165]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded) 2024-09-14T04:02:03.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:03 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:02:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[107399]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:05 smithi001 ceph-mon[112165]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:06.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:05 smithi012 ceph-mon[101180]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:06] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.51.0" 2024-09-14T04:02:06.981 INFO:teuthology.orchestra.run.smithi001.stdout:true 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:alertmanager.a smithi001 *:9093,9094 running (12m) 16s ago 20m 21.0M - 0.27.0 11f11916f8cd 4117f985a870 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:grafana.a smithi012 *:3000 running (12m) 49s ago 20m 84.8M - 10.4.0 c8b91775d855 d361d041a0ad 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi001.ytjgbj smithi001 running (19m) 16s ago 19m 25.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f 1c6577523f84 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:mds.foo.smithi012.nnhswt smithi012 running (19m) 49s ago 19m 17.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f ca7574dfb17e 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.x smithi012 *:8443,9283,8765 running (14m) 49s ago 24m 459M - 19.3.0-4907-ga2782315 8801d2529e43 23caad2752d8 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:mgr.y smithi001 *:8443,9283,8765 running (15m) 16s ago 26m 523M - 19.3.0-4907-ga2782315 8801d2529e43 5403daa74901 2024-09-14T04:02:07.571 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a smithi001 running (11m) 16s ago 26m 64.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 36cdffa077fb 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b smithi012 running (9m) 49s ago 24m 43.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 f246dcb03561 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c smithi001 running (10m) 16s ago 24m 50.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 a4ab7249719a 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.a smithi001 *:9100 running (14m) 16s ago 20m 12.3M - 1.7.0 72c9c2088986 6234cc31ec43 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:node-exporter.b smithi012 *:9100 running (13m) 49s ago 20m 11.1M - 1.7.0 72c9c2088986 a27184543b46 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 smithi001 running (5m) 16s ago 24m 495M 2481M 19.3.0-4907-ga2782315 8801d2529e43 6ed72c8797ba 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 smithi001 running (4m) 16s ago 23m 337M 2481M 19.3.0-4907-ga2782315 8801d2529e43 dc7254587752 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 smithi001 running (20s) 16s ago 23m 11.9M 2481M 19.3.0-4907-ga2782315 8801d2529e43 35b8e87dea9d 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 smithi001 running (22m) 16s ago 22m 1284M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f b312c78d4a2e 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 smithi012 running (22m) 49s ago 22m 603M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 8cf884c205fc 2024-09-14T04:02:07.572 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 smithi012 running (22m) 49s ago 22m 936M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f c828128eebc4 2024-09-14T04:02:07.573 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 smithi012 running (21m) 49s ago 21m 563M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f 31884087247e 2024-09-14T04:02:07.573 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 smithi012 running (21m) 49s ago 21m 548M 2481M 18.2.4-845-ge4e227b3 67ddd33fa72f e39770619660 2024-09-14T04:02:07.573 INFO:teuthology.orchestra.run.smithi001.stdout:prometheus.a smithi012 *:9095 running (13m) 49s ago 20m 51.9M - 2.51.0 1d3b7f56885b ebbb50c30a92 2024-09-14T04:02:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:07 smithi012 ceph-mon[101180]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:07 smithi012 ceph-mon[101180]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:07 smithi012 ceph-mon[101180]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[107399]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[107399]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[107399]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[112165]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[112165]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:07 smithi001 ceph-mon[112165]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:07.923 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:02:07.923 INFO:teuthology.orchestra.run.smithi001.stdout: "mon": { 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr": { 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "osd": { 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 5, 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "mds": { 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:02:07.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:02:07.925 INFO:teuthology.orchestra.run.smithi001.stdout: "overall": { 2024-09-14T04:02:07.925 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 7, 2024-09-14T04:02:07.925 INFO:teuthology.orchestra.run.smithi001.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T04:02:07.925 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:02:07.925 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:02:08.222 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:02:08.222 INFO:teuthology.orchestra.run.smithi001.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:02:08.223 INFO:teuthology.orchestra.run.smithi001.stdout: "in_progress": true, 2024-09-14T04:02:08.223 INFO:teuthology.orchestra.run.smithi001.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-09-14T04:02:08.223 INFO:teuthology.orchestra.run.smithi001.stdout: "services_complete": [], 2024-09-14T04:02:08.223 INFO:teuthology.orchestra.run.smithi001.stdout: "progress": "3/8 daemons upgraded", 2024-09-14T04:02:08.223 INFO:teuthology.orchestra.run.smithi001.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:02:08.224 INFO:teuthology.orchestra.run.smithi001.stdout: "is_paused": false 2024-09-14T04:02:08.224 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:02:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:08 smithi012 ceph-mon[101180]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3617111604' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:08 smithi012 ceph-mon[101180]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[107399]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3617111604' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[107399]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[112165]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3617111604' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:08 smithi001 ceph-mon[112165]: from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:09 smithi012 ceph-mon[101180]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:09 smithi001 ceph-mon[107399]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:09 smithi001 ceph-mon[112165]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:10 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:02:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:10 smithi012 ceph-mon[101180]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:02:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:10 smithi012 ceph-mon[101180]: Upgrade: osd.3 is safe to restart 2024-09-14T04:02:10.618 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:02:10.618 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[107399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:02:10.619 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[107399]: Upgrade: osd.3 is safe to restart 2024-09-14T04:02:10.619 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:02:10.619 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[112165]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:02:10.619 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:10 smithi001 ceph-mon[112165]: Upgrade: osd.3 is safe to restart 2024-09-14T04:02:11.568 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:11 smithi001 ceph-mon[107399]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:11.569 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:11 smithi001 ceph-mon[112165]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:11 smithi012 ceph-mon[101180]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[107399]: Upgrade: Updating osd.3 2024-09-14T04:02:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:02:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[107399]: Deploying daemon osd.3 on smithi001 2024-09-14T04:02:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[112165]: Upgrade: Updating osd.3 2024-09-14T04:02:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:02:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:12 smithi001 ceph-mon[112165]: Deploying daemon osd.3 on smithi001 2024-09-14T04:02:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:12 smithi012 ceph-mon[101180]: Upgrade: Updating osd.3 2024-09-14T04:02:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:02:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:12 smithi012 ceph-mon[101180]: Deploying daemon osd.3 on smithi001 2024-09-14T04:02:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:13 smithi001 ceph-mon[112165]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:13 smithi001 ceph-mon[107399]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:13 smithi012 ceph-mon[101180]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:14.887 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:14 smithi001 systemd[1]: Stopping Ceph osd.3 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:02:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:14 smithi001 ceph-mon[112165]: osd.3 marked itself down and dead 2024-09-14T04:02:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:14 smithi001 ceph-mon[107399]: osd.3 marked itself down and dead 2024-09-14T04:02:15.221 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[64135]: 2024-09-14T04:02:14.884+0000 7f3557a1d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:02:15.222 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[64135]: 2024-09-14T04:02:14.884+0000 7f3557a1d640 -1 osd.3 126 *** Got signal Terminated *** 2024-09-14T04:02:15.222 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[64135]: 2024-09-14T04:02:14.884+0000 7f3557a1d640 -1 osd.3 126 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:02:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:14 smithi012 ceph-mon[101180]: osd.3 marked itself down and dead 2024-09-14T04:02:15.677 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:15 smithi001 podman[141181]: 2024-09-14 04:02:15.395967803 +0000 UTC m=+0.637152270 container died b312c78d4a2e76e5afe881684aceb12328c5ee510b9e6732215ede35f387cb05 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T04:02:15.953 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:15 smithi001 podman[141181]: 2024-09-14 04:02:15.77091954 +0000 UTC m=+1.012103999 container cleanup b312c78d4a2e76e5afe881684aceb12328c5ee510b9e6732215ede35f387cb05 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b) 2024-09-14T04:02:15.954 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:15 smithi001 bash[141181]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3 2024-09-14T04:02:16.221 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141202]: 2024-09-14 04:02:16.064212985 +0000 UTC m=+0.664722719 container remove b312c78d4a2e76e5afe881684aceb12328c5ee510b9e6732215ede35f387cb05 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, RELEASE=reef-e4e227b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:02:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:02:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[107399]: osdmap e127: 8 total, 7 up, 8 in 2024-09-14T04:02:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[107399]: pgmap v387: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[107399]: osdmap e128: 8 total, 7 up, 8 in 2024-09-14T04:02:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:02:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[112165]: osdmap e127: 8 total, 7 up, 8 in 2024-09-14T04:02:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[112165]: pgmap v387: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:15 smithi001 ceph-mon[112165]: osdmap e128: 8 total, 7 up, 8 in 2024-09-14T04:02:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:15 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:02:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:15 smithi012 ceph-mon[101180]: osdmap e127: 8 total, 7 up, 8 in 2024-09-14T04:02:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:15 smithi012 ceph-mon[101180]: pgmap v387: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:16.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:15 smithi012 ceph-mon[101180]: osdmap e128: 8 total, 7 up, 8 in 2024-09-14T04:02:16.676 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.390011297 +0000 UTC m=+0.106440196 container create fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-14T04:02:16.676 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.300223252 +0000 UTC m=+0.016652155 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:02:16.677 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.592746989 +0000 UTC m=+0.309175888 container init fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231) 2024-09-14T04:02:16.677 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.598454711 +0000 UTC m=+0.314883619 container start fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-14T04:02:16.677 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.675790999 +0000 UTC m=+0.392219897 container attach fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:02:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:16] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.51.0" 2024-09-14T04:02:16.971 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:16 smithi001 podman[141296]: 2024-09-14 04:02:16.796652689 +0000 UTC m=+0.513081594 container died fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T04:02:17.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[107399]: pgmap v389: 73 pgs: 10 active+undersized, 13 peering, 2 stale+active+clean, 1 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:17.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-14T04:02:17.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:02:17.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[112165]: pgmap v389: 73 pgs: 10 active+undersized, 13 peering, 2 stale+active+clean, 1 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:17.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-14T04:02:17.390 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:17 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:02:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:17 smithi012 ceph-mon[101180]: pgmap v389: 73 pgs: 10 active+undersized, 13 peering, 2 stale+active+clean, 1 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:17 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-14T04:02:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:17 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:02:17.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 podman[141339]: 2024-09-14 04:02:17.388638367 +0000 UTC m=+0.585350372 container remove fb3c71f0d4d1211838aeb0376618624f77c1c2514050921705c16c06ee87112c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T04:02:17.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-14T04:02:17.722 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 systemd[1]: Stopped Ceph osd.3 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:02:17.722 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.3.service: Consumed 29min 58.388s CPU time. 2024-09-14T04:02:17.722 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 systemd[1]: Starting Ceph osd.3 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:02:18.170 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 podman[141444]: 2024-09-14 04:02:17.72527978 +0000 UTC m=+0.017098822 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:02:18.170 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:17 smithi001 podman[141444]: 2024-09-14 04:02:17.823169108 +0000 UTC m=+0.114988149 container create 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-14T04:02:18.423 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 podman[141444]: 2024-09-14 04:02:18.275998314 +0000 UTC m=+0.567817356 container init 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:02:18.423 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 podman[141444]: 2024-09-14 04:02:18.282154183 +0000 UTC m=+0.573973225 container start 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231) 2024-09-14T04:02:18.423 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 podman[141444]: 2024-09-14 04:02:18.359097007 +0000 UTC m=+0.650916058 container attach 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=main-a278231, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:02:18.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:02:18.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 bash[141444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:02:18.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:02:18.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 bash[141444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:02:19.188 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:02:19.188 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 bash[141444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:02:19.188 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T04:02:19.188 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:18 smithi001 bash[141444]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T04:02:19.189 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T04:02:19.189 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 bash[141444]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T04:02:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-mon[107399]: pgmap v390: 73 pgs: 14 active+undersized, 13 peering, 1 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-mon[112165]: pgmap v390: 73 pgs: 14 active+undersized, 13 peering, 1 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:19.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T04:02:19.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 bash[141444]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T04:02:19.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T04:02:19.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 bash[141444]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T04:02:19.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:02:19.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 bash[141444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:02:19.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate[141458]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T04:02:19.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 bash[141444]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T04:02:19.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 podman[141444]: 2024-09-14 04:02:19.217532563 +0000 UTC m=+1.509351605 container died 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T04:02:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:19 smithi012 ceph-mon[101180]: pgmap v390: 73 pgs: 14 active+undersized, 13 peering, 1 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-14T04:02:19.971 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:19 smithi001 podman[141608]: 2024-09-14 04:02:19.82386599 +0000 UTC m=+0.595529130 container remove 726b80ad98b3be01629a02d6874c5b2616e4c93f00b0aae6271fdd85ccf3fa89 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:02:20.288 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 podman[141673]: 2024-09-14 04:02:20.033112356 +0000 UTC m=+0.109169346 container create e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:02:20.288 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 podman[141673]: 2024-09-14 04:02:19.940621621 +0000 UTC m=+0.016678619 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:02:20.288 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 podman[141673]: 2024-09-14 04:02:20.260977737 +0000 UTC m=+0.337034735 container init e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T04:02:20.288 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 podman[141673]: 2024-09-14 04:02:20.265610793 +0000 UTC m=+0.341667795 container start e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:02:20.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-osd[141712]: -- 172.21.15.1:0/4078770610 <== mon.2 v2:172.21.15.1:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55805b937a40 con 0x55805c706400 2024-09-14T04:02:20.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 bash[141673]: e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 2024-09-14T04:02:20.721 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:20 smithi001 systemd[1]: Started Ceph osd.3 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:02:21.168 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.168 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:21.169 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.169 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.169 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.169 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:21.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:21 smithi012 ceph-mon[101180]: pgmap v391: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:22.157 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:21 smithi001 ceph-mon[107399]: pgmap v391: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:22.157 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:21 smithi001 ceph-mon[112165]: pgmap v391: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:22.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:02:22.155+0000 7feff0aee640 -1 Falling back to public interface 2024-09-14T04:02:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:22 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-14T04:02:23.181 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:22 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-14T04:02:23.181 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:22 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-14T04:02:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:23 smithi001 ceph-mon[107399]: pgmap v392: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:23 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:23.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:23 smithi001 ceph-mon[112165]: pgmap v392: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:23.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:23 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:24.317 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:23 smithi012 ceph-mon[101180]: pgmap v392: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:24.317 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:23 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:02:25.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.450 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:25 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:25 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:26 smithi001 ceph-mon[107399]: pgmap v393: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:26 smithi001 ceph-mon[112165]: pgmap v393: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:26 smithi012 ceph-mon[101180]: pgmap v393: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:26.790 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:26] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T04:02:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:27 smithi012 ceph-mon[101180]: pgmap v394: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:27.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:27 smithi001 ceph-mon[107399]: pgmap v394: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:27.669 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:27 smithi001 ceph-mon[112165]: pgmap v394: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: Upgrade: Finalizing container_image settings 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:02:29.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.474 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.475 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.476 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: Upgrade: Finalizing container_image settings 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:02:29.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:02:29.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.480 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:29 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: Upgrade: Finalizing container_image settings 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:02:29.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:29.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:29 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[107399]: Upgrade: Complete! 2024-09-14T04:02:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[107399]: pgmap v395: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[112165]: Upgrade: Complete! 2024-09-14T04:02:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[112165]: pgmap v395: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:30.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:30 smithi012 ceph-mon[101180]: Upgrade: Complete! 2024-09-14T04:02:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:30 smithi012 ceph-mon[101180]: pgmap v395: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:30 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:30.971 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:02:30.615+0000 7feff0aee640 -1 osd.3 126 log_to_monitors true 2024-09-14T04:02:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:31 smithi001 ceph-mon[107399]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:31 smithi001 ceph-mon[107399]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:31 smithi001 ceph-mon[112165]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:31.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:31 smithi001 ceph-mon[112165]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:31.472 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:02:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:02:31.193+0000 7fefe8898640 -1 osd.3 126 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:02:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:31 smithi012 ceph-mon[101180]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:31 smithi012 ceph-mon[101180]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:02:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[107399]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:02:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[107399]: osdmap e129: 8 total, 7 up, 8 in 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[107399]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[107399]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[107399]: pgmap v397: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[112165]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[112165]: osdmap e129: 8 total, 7 up, 8 in 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[112165]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[112165]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:32 smithi001 ceph-mon[112165]: pgmap v397: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:32 smithi012 ceph-mon[101180]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:02:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:32 smithi012 ceph-mon[101180]: osdmap e129: 8 total, 7 up, 8 in 2024-09-14T04:02:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:32 smithi012 ceph-mon[101180]: from='osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:32 smithi012 ceph-mon[101180]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:02:32.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:32 smithi012 ceph-mon[101180]: pgmap v397: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[107399]: osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471] boot 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[107399]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[107399]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T04:02:33.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:02:33.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[112165]: osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471] boot 2024-09-14T04:02:33.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[112165]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T04:02:33.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:02:33.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:33 smithi001 ceph-mon[112165]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T04:02:33.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:33 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:02:33.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:33 smithi012 ceph-mon[101180]: osd.3 [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471] boot 2024-09-14T04:02:33.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:33 smithi012 ceph-mon[101180]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T04:02:33.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:02:33.498 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:33 smithi012 ceph-mon[101180]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T04:02:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:34 smithi001 ceph-mon[107399]: pgmap v400: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:34 smithi001 ceph-mon[112165]: pgmap v400: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:34 smithi012 ceph-mon[101180]: pgmap v400: 73 pgs: 25 active+undersized, 3 active+undersized+degraded, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[107399]: pgmap v401: 73 pgs: 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:35 smithi001 ceph-mon[112165]: pgmap v401: 73 pgs: 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:36.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:35 smithi012 ceph-mon[101180]: pgmap v401: 73 pgs: 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:02:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:36] "GET /metrics HTTP/1.1" 200 33880 "" "Prometheus/2.51.0" 2024-09-14T04:02:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:37 smithi012 ceph-mon[101180]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:37 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-09-14T04:02:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:37 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:02:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[107399]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-09-14T04:02:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:02:37.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[112165]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:37.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-09-14T04:02:37.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:37 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:02:39.300 DEBUG:teuthology.parallel:result is None 2024-09-14T04:02:39.300 INFO:teuthology.run_tasks:Running task print... 2024-09-14T04:02:39.309 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-14T04:02:39.310 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T04:02:39.331 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T04:02:39.332 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T04:02:39.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T04:02:39.545 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:39 smithi012 ceph-mon[101180]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:39 smithi012 ceph-mon[101180]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:39.666 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:39 smithi001 ceph-mon[107399]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:39.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:39 smithi001 ceph-mon[107399]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:39.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:39 smithi001 ceph-mon[112165]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:39.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:39 smithi001 ceph-mon[112165]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:41.543 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:02:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:41 smithi012 ceph-mon[101180]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:41 smithi001 ceph-mon[107399]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:41 smithi001 ceph-mon[112165]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:42.124 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":41,"flags":0,"active_gid":34104,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6800","nonce":1949317846},{"type":"v1","addr":"172.21.15.1:6801","nonce":1949317846}]},"active_addr":"172.21.15.1:6801/1949317846","active_change":"2024-09-14T03:50:48.967736+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":24943,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":86,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":1718382881}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":988146407}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":3687297502}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.1:0","nonce":2491102211}]}]} 2024-09-14T04:02:42.129 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T04:02:42.129 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T04:02:42.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:02:42.344 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:42.465 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2155654296' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:02:42.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2155654296' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:02:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2155654296' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:02:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:43 smithi001 ceph-mon[107399]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:43 smithi001 ceph-mon[112165]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:43 smithi012 ceph-mon[101180]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:44.243 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:02:44.244 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":131,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T04:02:33.174693+0000","last_up_change":"2024-09-14T04:02:32.172888+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":102,"up_thru":130,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6803","nonce":1278848157}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6805","nonce":1278848157}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6809","nonce":1278848157}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6807","nonce":1278848157}]},"public_addr":"172.21.15.1:6803/1278848157","cluster_addr":"172.21.15.1:6805/1278848157","heartbeat_back_addr":"172.21.15.1:6809/1278848157","heartbeat_front_addr":"172.21.15.1:6807/1278848157","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":104,"up_from":108,"up_thru":130,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6811","nonce":2795716422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6813","nonce":2795716422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6817","nonce":2795716422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6815","nonce":2795716422}]},"public_addr":"172.21.15.1:6811/2795716422","cluster_addr":"172.21.15.1:6813/2795716422","heartbeat_back_addr":"172.21.15.1:6817/2795716422","heartbeat_front_addr":"172.21.15.1:6815/2795716422","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":121,"up_from":125,"up_thru":130,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6819","nonce":4111579272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6821","nonce":4111579272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6825","nonce":4111579272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6823","nonce":4111579272}]},"public_addr":"172.21.15.1:6819/4111579272","cluster_addr":"172.21.15.1:6821/4111579272","heartbeat_back_addr":"172.21.15.1:6825/4111579272","heartbeat_front_addr":"172.21.15.1:6823/4111579272","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":130,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6827","nonce":3997365471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6829","nonce":3997365471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6833","nonce":3997365471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6831","nonce":3997365471}]},"public_addr":"172.21.15.1:6827/3997365471","cluster_addr":"172.21.15.1:6829/3997365471","heartbeat_back_addr":"172.21.15.1:6833/3997365471","heartbeat_front_addr":"172.21.15.1:6831/3997365471","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":127,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":121},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.5","mappings":[{"from":3,"to":0}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.13","mappings":[{"from":7,"to":0}]},{"pgid":"3.1a","mappings":[{"from":4,"to":0}]},{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":3,"to":0}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:6800/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/1739684728":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.12:0/2389061674":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.12:6834/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/4171009491":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2949027530":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/739412744":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/2817796360":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/1135126809":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/2685165922":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/41633275":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/416113000":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6801/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/3674234575":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.12:6835/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/2249577072":"2024-09-15T03:50:48.967020+0000","172.21.15.12:0/3549839308":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.12:0/3359585252":"2024-09-15T03:47:16.144756+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:02:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:44 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3146610722' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:44 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3146610722' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:44 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3146610722' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:44.859 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T04:02:44.859 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:02:45.073 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:46.196 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:46.196 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:46 smithi001 ceph-mon[107399]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:46.196 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:46.196 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:46 smithi001 ceph-mon[112165]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:46 smithi012 ceph-mon[101180]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:46.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:46] "GET /metrics HTTP/1.1" 200 33880 "" "Prometheus/2.51.0" 2024-09-14T04:02:46.935 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:02:46.935 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":131,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T04:02:33.174693+0000","last_up_change":"2024-09-14T04:02:32.172888+0000","last_in_change":"2024-09-14T03:40:29.971756+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":102,"up_thru":130,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6803","nonce":1278848157}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6805","nonce":1278848157}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6809","nonce":1278848157}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":1278848157},{"type":"v1","addr":"172.21.15.1:6807","nonce":1278848157}]},"public_addr":"172.21.15.1:6803/1278848157","cluster_addr":"172.21.15.1:6805/1278848157","heartbeat_back_addr":"172.21.15.1:6809/1278848157","heartbeat_front_addr":"172.21.15.1:6807/1278848157","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":104,"up_from":108,"up_thru":130,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6811","nonce":2795716422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6813","nonce":2795716422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6817","nonce":2795716422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6815","nonce":2795716422}]},"public_addr":"172.21.15.1:6811/2795716422","cluster_addr":"172.21.15.1:6813/2795716422","heartbeat_back_addr":"172.21.15.1:6817/2795716422","heartbeat_front_addr":"172.21.15.1:6815/2795716422","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":121,"up_from":125,"up_thru":130,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6819","nonce":4111579272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6821","nonce":4111579272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6825","nonce":4111579272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":4111579272},{"type":"v1","addr":"172.21.15.1:6823","nonce":4111579272}]},"public_addr":"172.21.15.1:6819/4111579272","cluster_addr":"172.21.15.1:6821/4111579272","heartbeat_back_addr":"172.21.15.1:6825/4111579272","heartbeat_front_addr":"172.21.15.1:6823/4111579272","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":130,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6827","nonce":3997365471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6829","nonce":3997365471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6833","nonce":3997365471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6831","nonce":3997365471}]},"public_addr":"172.21.15.1:6827/3997365471","cluster_addr":"172.21.15.1:6829/3997365471","heartbeat_back_addr":"172.21.15.1:6833/3997365471","heartbeat_front_addr":"172.21.15.1:6831/3997365471","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6809","nonce":2995357883}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6811","nonce":2995357883}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6815","nonce":2995357883}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":2995357883},{"type":"v1","addr":"172.21.15.12:6813","nonce":2995357883}]},"public_addr":"172.21.15.12:6809/2995357883","cluster_addr":"172.21.15.12:6811/2995357883","heartbeat_back_addr":"172.21.15.12:6815/2995357883","heartbeat_front_addr":"172.21.15.12:6813/2995357883","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":127,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":121},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.5","mappings":[{"from":3,"to":0}]},{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.13","mappings":[{"from":7,"to":0}]},{"pgid":"3.1a","mappings":[{"from":4,"to":0}]},{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":3,"to":0}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:6800/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/1739684728":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.12:0/2389061674":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.12:6834/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/4171009491":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2949027530":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/739412744":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/2817796360":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/1135126809":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/2685165922":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/41633275":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/416113000":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6801/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/3674234575":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.12:6835/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/2249577072":"2024-09-15T03:50:48.967020+0000","172.21.15.12:0/3549839308":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.12:0/3359585252":"2024-09-15T03:47:16.144756+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:02:47.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1044788884' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:47.312 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1044788884' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1044788884' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:47.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T04:02:47.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T04:02:47.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T04:02:47.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T04:02:47.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T04:02:47.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T04:02:47.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T04:02:47.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T04:02:48.101 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.101 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.103 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.103 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.104 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.108 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.109 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.115 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:48 smithi012 ceph-mon[101180]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:48 smithi001 ceph-mon[107399]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:48 smithi001 ceph-mon[112165]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:49.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:49 smithi001 ceph-mon[107399]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:49.408 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:49 smithi001 ceph-mon[112165]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:49 smithi012 ceph-mon[101180]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"} : dispatch 2024-09-14T04:02:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:50.598 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"} : dispatch 2024-09-14T04:02:50.598 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:50.598 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:50.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"} : dispatch 2024-09-14T04:02:50.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:02:50.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:51.417 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"}]': finished 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[107399]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[107399]: pgmap v410: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[107399]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"}]': finished 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[112165]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[112165]: pgmap v410: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:51.418 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:51 smithi001 ceph-mon[112165]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T04:02:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1c"}]': finished 2024-09-14T04:02:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:51 smithi012 ceph-mon[101180]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T04:02:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:51 smithi012 ceph-mon[101180]: pgmap v410: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:51 smithi012 ceph-mon[101180]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T04:02:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:53 smithi012 ceph-mon[101180]: pgmap v412: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:53 smithi001 ceph-mon[107399]: pgmap v412: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:53 smithi001 ceph-mon[112165]: pgmap v412: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:55 smithi012 ceph-mon[101180]: pgmap v413: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:55 smithi001 ceph-mon[107399]: pgmap v413: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:55.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:55 smithi001 ceph-mon[112165]: pgmap v413: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:02:55.750 INFO:teuthology.orchestra.run.smithi001.stdout:558345748486 2024-09-14T04:02:55.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T04:02:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:02:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:02:56] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.51.0" 2024-09-14T04:02:57.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:57 smithi001 ceph-mon[107399]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:02:57.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:57 smithi001 ceph-mon[112165]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:02:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:57 smithi012 ceph-mon[101180]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:02:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:02:59 smithi012 ceph-mon[101180]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:02:59.627 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:02:59 smithi001 ceph-mon[107399]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:02:59.627 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:02:59 smithi001 ceph-mon[112165]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:00.601 INFO:teuthology.orchestra.run.smithi001.stdout:536870912014 2024-09-14T04:03:00.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T04:03:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:01 smithi001 ceph-mon[107399]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:01 smithi001 ceph-mon[112165]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:01 smithi012 ceph-mon[101180]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:02.118 INFO:teuthology.orchestra.run.smithi001.stdout:167503724823 2024-09-14T04:03:02.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T04:03:03.453 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:03 smithi001 ceph-mon[107399]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:03.453 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:03 smithi001 ceph-mon[112165]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:03.476 INFO:teuthology.orchestra.run.smithi001.stdout:201863463187 2024-09-14T04:03:03.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T04:03:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:03 smithi012 ceph-mon[101180]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:03.658 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:04.063 INFO:teuthology.orchestra.run.smithi001.stdout:236223201548 2024-09-14T04:03:04.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T04:03:04.983 INFO:teuthology.orchestra.run.smithi001.stdout:133143986459 2024-09-14T04:03:04.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T04:03:05.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:05.423 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:05.444 INFO:teuthology.orchestra.run.smithi001.stdout:463856468031 2024-09-14T04:03:05.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T04:03:05.817 INFO:teuthology.orchestra.run.smithi001.stdout:438086664265 2024-09-14T04:03:05.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T04:03:06.070 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:05 smithi001 ceph-mon[107399]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:06.071 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:05 smithi001 ceph-mon[112165]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:05 smithi012 ceph-mon[101180]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:06.533 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:06.533 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:06.580 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:06.938 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:06] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.51.0" 2024-09-14T04:03:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:07 smithi001 ceph-mon[107399]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:07 smithi001 ceph-mon[112165]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:07.511 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:07 smithi012 ceph-mon[101180]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:03:07.855 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:09 smithi001 ceph-mon[107399]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:09 smithi001 ceph-mon[112165]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:09 smithi012 ceph-mon[101180]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:10.912 INFO:teuthology.orchestra.run.smithi001.stdout:236223201550 2024-09-14T04:03:11.168 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2743628940' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:03:11.168 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2743628940' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:03:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2743628940' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:03:12.015 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:11 smithi001 ceph-mon[107399]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:12.015 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:11 smithi001 ceph-mon[112165]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:12.043 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201548 got 236223201550 for osd.7 2024-09-14T04:03:12.043 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:11 smithi012 ceph-mon[101180]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:13 smithi001 ceph-mon[112165]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:13 smithi001 ceph-mon[107399]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:13 smithi012 ceph-mon[101180]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:14.881 INFO:teuthology.orchestra.run.smithi001.stdout:201863463189 2024-09-14T04:03:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4185824542' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:03:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4185824542' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:03:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4185824542' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:03:15.423 INFO:teuthology.orchestra.run.smithi001.stdout:167503724825 2024-09-14T04:03:15.683 INFO:teuthology.orchestra.run.smithi001.stdout:558345748490 2024-09-14T04:03:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2026067947' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:03:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2356820139' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:03:16.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[107399]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:16.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2026067947' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:03:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2356820139' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:03:16.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:15 smithi001 ceph-mon[112165]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2026067947' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:03:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2356820139' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:03:16.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:15 smithi012 ceph-mon[101180]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:16.366 INFO:tasks.cephadm.ceph_manager.ceph:need seq 558345748486 got 558345748490 for osd.3 2024-09-14T04:03:16.366 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:16] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.51.0" 2024-09-14T04:03:17.207 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724823 got 167503724825 for osd.5 2024-09-14T04:03:17.207 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:17 smithi001 ceph-mon[107399]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:17 smithi001 ceph-mon[112165]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:17 smithi012 ceph-mon[101180]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:17.641 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463187 got 201863463189 for osd.6 2024-09-14T04:03:17.642 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:18.084 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:18.869 INFO:teuthology.orchestra.run.smithi001.stdout:438086664268 2024-09-14T04:03:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3251497774' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:03:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3251497774' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:03:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3251497774' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:03:19.809 INFO:tasks.cephadm.ceph_manager.ceph:need seq 438086664265 got 438086664268 for osd.0 2024-09-14T04:03:19.810 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:20.058 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:19 smithi001 ceph-mon[112165]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:20.058 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:20.058 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:19 smithi001 ceph-mon[107399]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:20.058 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:19 smithi012 ceph-mon[101180]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:20.358 INFO:teuthology.orchestra.run.smithi001.stdout:133143986462 2024-09-14T04:03:20.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1631169013' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:03:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1631169013' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:03:21.303 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986459 got 133143986462 for osd.4 2024-09-14T04:03:21.303 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1631169013' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:03:21.732 INFO:teuthology.orchestra.run.smithi001.stdout:536870912018 2024-09-14T04:03:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:21 smithi001 ceph-mon[107399]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3062817285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:03:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:21 smithi001 ceph-mon[112165]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3062817285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:03:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:21 smithi012 ceph-mon[101180]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3062817285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:03:22.840 INFO:teuthology.orchestra.run.smithi001.stdout:463856468035 2024-09-14T04:03:23.103 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:22 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3262713755' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:03:23.103 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:22 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3262713755' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:03:23.124 INFO:tasks.cephadm.ceph_manager.ceph:need seq 536870912014 got 536870912018 for osd.2 2024-09-14T04:03:23.125 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:22 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3262713755' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:03:23.768 INFO:tasks.cephadm.ceph_manager.ceph:need seq 463856468031 got 463856468035 for osd.1 2024-09-14T04:03:23.768 DEBUG:teuthology.parallel:result is None 2024-09-14T04:03:23.768 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T04:03:23.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:03:23.985 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:24.106 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:23 smithi001 ceph-mon[107399]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:24.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:23 smithi001 ceph-mon[112165]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:23 smithi012 ceph-mon[101180]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:25 smithi001 ceph-mon[107399]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:25.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:25 smithi001 ceph-mon[112165]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:25 smithi012 ceph-mon[101180]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:25.883 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:03:25.884 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:03:26.564 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:26 smithi001 ceph-mon[107399]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:26.565 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:26 smithi001 ceph-mon[112165]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:26.590 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":428,"stamp":"2024-09-14T04:03:25.283559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771138,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2591920,"kb_used_data":14468,"kb_used_omap":28,"kb_used_meta":2577315,"kb_avail":747369296,"statfs":{"total":767960285184,"available":765306159104,"internally_reserved":0,"allocated":14815232,"data_stored":8544655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29302,"internal_metadata":2639170954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003133"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918602+0000","last_change":"2024-09-14T04:02:33.186418+0000","last_active":"2024-09-14T04:02:51.918602+0000","last_peered":"2024-09-14T04:02:51.918602+0000","last_clean":"2024-09-14T04:02:51.918602+0000","last_became_active":"2024-09-14T04:02:33.186052+0000","last_became_peered":"2024-09-14T04:02:33.186052+0000","last_unstale":"2024-09-14T04:02:51.918602+0000","last_undegraded":"2024-09-14T04:02:51.918602+0000","last_fullsized":"2024-09-14T04:02:51.918602+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:04.240828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":89,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342633+0000","last_change":"2024-09-14T04:01:59.055661+0000","last_active":"2024-09-14T04:02:51.342633+0000","last_peered":"2024-09-14T04:02:51.342633+0000","last_clean":"2024-09-14T04:02:51.342633+0000","last_became_active":"2024-09-14T04:01:59.055301+0000","last_became_peered":"2024-09-14T04:01:59.055301+0000","last_unstale":"2024-09-14T04:02:51.342633+0000","last_undegraded":"2024-09-14T04:02:51.342633+0000","last_fullsized":"2024-09-14T04:02:51.342633+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:34:27.575772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338958+0000","last_change":"2024-09-14T04:02:33.189718+0000","last_active":"2024-09-14T04:02:50.338958+0000","last_peered":"2024-09-14T04:02:50.338958+0000","last_clean":"2024-09-14T04:02:50.338958+0000","last_became_active":"2024-09-14T04:02:33.189342+0000","last_became_peered":"2024-09-14T04:02:33.189342+0000","last_unstale":"2024-09-14T04:02:50.338958+0000","last_undegraded":"2024-09-14T04:02:50.338958+0000","last_fullsized":"2024-09-14T04:02:50.338958+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:51:37.536543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342588+0000","last_change":"2024-09-14T04:02:33.196773+0000","last_active":"2024-09-14T04:02:51.342588+0000","last_peered":"2024-09-14T04:02:51.342588+0000","last_clean":"2024-09-14T04:02:51.342588+0000","last_became_active":"2024-09-14T04:02:33.196447+0000","last_became_peered":"2024-09-14T04:02:33.196447+0000","last_unstale":"2024-09-14T04:02:51.342588+0000","last_undegraded":"2024-09-14T04:02:51.342588+0000","last_fullsized":"2024-09-14T04:02:51.342588+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:51.157998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342796+0000","last_change":"2024-09-14T04:02:33.198492+0000","last_active":"2024-09-14T04:02:51.342796+0000","last_peered":"2024-09-14T04:02:51.342796+0000","last_clean":"2024-09-14T04:02:51.342796+0000","last_became_active":"2024-09-14T04:02:33.197804+0000","last_became_peered":"2024-09-14T04:02:33.197804+0000","last_unstale":"2024-09-14T04:02:51.342796+0000","last_undegraded":"2024-09-14T04:02:51.342796+0000","last_fullsized":"2024-09-14T04:02:51.342796+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:37.842086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342325+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:02:50.342325+0000","last_peered":"2024-09-14T04:02:50.342325+0000","last_clean":"2024-09-14T04:02:50.342325+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:02:50.342325+0000","last_undegraded":"2024-09-14T04:02:50.342325+0000","last_fullsized":"2024-09-14T04:02:50.342325+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343001+0000","last_change":"2024-09-14T03:57:55.641587+0000","last_active":"2024-09-14T04:02:51.343001+0000","last_peered":"2024-09-14T04:02:51.343001+0000","last_clean":"2024-09-14T04:02:51.343001+0000","last_became_active":"2024-09-14T03:57:55.640967+0000","last_became_peered":"2024-09-14T03:57:55.640967+0000","last_unstale":"2024-09-14T04:02:51.343001+0000","last_undegraded":"2024-09-14T04:02:51.343001+0000","last_fullsized":"2024-09-14T04:02:51.343001+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:05.024573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":103,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.181388+0000","last_change":"2024-09-14T03:57:07.969661+0000","last_active":"2024-09-14T04:02:33.181388+0000","last_peered":"2024-09-14T04:02:33.181388+0000","last_clean":"2024-09-14T04:02:33.181388+0000","last_became_active":"2024-09-14T03:57:07.969236+0000","last_became_peered":"2024-09-14T03:57:07.969236+0000","last_unstale":"2024-09-14T04:02:33.181388+0000","last_undegraded":"2024-09-14T04:02:33.181388+0000","last_fullsized":"2024-09-14T04:02:33.181388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:28.685717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.919228+0000","last_change":"2024-09-14T03:57:07.978483+0000","last_active":"2024-09-14T04:02:51.919228+0000","last_peered":"2024-09-14T04:02:51.919228+0000","last_clean":"2024-09-14T04:02:51.919228+0000","last_became_active":"2024-09-14T03:57:07.978076+0000","last_became_peered":"2024-09-14T03:57:07.978076+0000","last_unstale":"2024-09-14T04:02:51.919228+0000","last_undegraded":"2024-09-14T04:02:51.919228+0000","last_fullsized":"2024-09-14T04:02:51.919228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:33.126366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338345+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T04:02:50.338345+0000","last_peered":"2024-09-14T04:02:50.338345+0000","last_clean":"2024-09-14T04:02:50.338345+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T04:02:50.338345+0000","last_undegraded":"2024-09-14T04:02:50.338345+0000","last_fullsized":"2024-09-14T04:02:50.338345+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918439+0000","last_change":"2024-09-14T03:57:55.658263+0000","last_active":"2024-09-14T04:02:51.918439+0000","last_peered":"2024-09-14T04:02:51.918439+0000","last_clean":"2024-09-14T04:02:51.918439+0000","last_became_active":"2024-09-14T03:57:55.657817+0000","last_became_peered":"2024-09-14T03:57:55.657817+0000","last_unstale":"2024-09-14T04:02:51.918439+0000","last_undegraded":"2024-09-14T04:02:51.918439+0000","last_fullsized":"2024-09-14T04:02:51.918439+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:36.196909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341202+0000","last_change":"2024-09-14T04:02:33.196376+0000","last_active":"2024-09-14T04:02:50.341202+0000","last_peered":"2024-09-14T04:02:50.341202+0000","last_clean":"2024-09-14T04:02:50.341202+0000","last_became_active":"2024-09-14T04:02:33.195804+0000","last_became_peered":"2024-09-14T04:02:33.195804+0000","last_unstale":"2024-09-14T04:02:50.341202+0000","last_undegraded":"2024-09-14T04:02:50.341202+0000","last_fullsized":"2024-09-14T04:02:50.341202+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:58:02.150113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918423+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T04:02:51.918423+0000","last_peered":"2024-09-14T04:02:51.918423+0000","last_clean":"2024-09-14T04:02:51.918423+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T04:02:51.918423+0000","last_undegraded":"2024-09-14T04:02:51.918423+0000","last_fullsized":"2024-09-14T04:02:51.918423+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182266+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:02:33.182266+0000","last_peered":"2024-09-14T04:02:33.182266+0000","last_clean":"2024-09-14T04:02:33.182266+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:02:33.182266+0000","last_undegraded":"2024-09-14T04:02:33.182266+0000","last_fullsized":"2024-09-14T04:02:33.182266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.598290+0000","last_change":"2024-09-14T04:02:33.189804+0000","last_active":"2024-09-14T04:02:50.598290+0000","last_peered":"2024-09-14T04:02:50.598290+0000","last_clean":"2024-09-14T04:02:50.598290+0000","last_became_active":"2024-09-14T04:02:33.189427+0000","last_became_peered":"2024-09-14T04:02:33.189427+0000","last_unstale":"2024-09-14T04:02:50.598290+0000","last_undegraded":"2024-09-14T04:02:50.598290+0000","last_fullsized":"2024-09-14T04:02:50.598290+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:35.027998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342724+0000","last_change":"2024-09-14T03:57:55.640069+0000","last_active":"2024-09-14T04:02:51.342724+0000","last_peered":"2024-09-14T04:02:51.342724+0000","last_clean":"2024-09-14T04:02:51.342724+0000","last_became_active":"2024-09-14T03:57:55.639653+0000","last_became_peered":"2024-09-14T03:57:55.639653+0000","last_unstale":"2024-09-14T04:02:51.342724+0000","last_undegraded":"2024-09-14T04:02:51.342724+0000","last_fullsized":"2024-09-14T04:02:51.342724+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:16.579752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918366+0000","last_change":"2024-09-14T04:02:33.189648+0000","last_active":"2024-09-14T04:02:51.918366+0000","last_peered":"2024-09-14T04:02:51.918366+0000","last_clean":"2024-09-14T04:02:51.918366+0000","last_became_active":"2024-09-14T04:02:33.188956+0000","last_became_peered":"2024-09-14T04:02:33.188956+0000","last_unstale":"2024-09-14T04:02:51.918366+0000","last_undegraded":"2024-09-14T04:02:51.918366+0000","last_fullsized":"2024-09-14T04:02:51.918366+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:14.851661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.384607+0000","last_change":"2024-09-14T04:02:33.194407+0000","last_active":"2024-09-14T04:02:50.384607+0000","last_peered":"2024-09-14T04:02:50.384607+0000","last_clean":"2024-09-14T04:02:50.384607+0000","last_became_active":"2024-09-14T04:02:33.194049+0000","last_became_peered":"2024-09-14T04:02:33.194049+0000","last_unstale":"2024-09-14T04:02:50.384607+0000","last_undegraded":"2024-09-14T04:02:50.384607+0000","last_fullsized":"2024-09-14T04:02:50.384607+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:09.389524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182415+0000","last_change":"2024-09-14T03:57:55.680448+0000","last_active":"2024-09-14T04:02:33.182415+0000","last_peered":"2024-09-14T04:02:33.182415+0000","last_clean":"2024-09-14T04:02:33.182415+0000","last_became_active":"2024-09-14T03:57:55.680199+0000","last_became_peered":"2024-09-14T03:57:55.680199+0000","last_unstale":"2024-09-14T04:02:33.182415+0000","last_undegraded":"2024-09-14T04:02:33.182415+0000","last_fullsized":"2024-09-14T04:02:33.182415+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:03.237323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340655+0000","last_change":"2024-09-14T04:01:59.052174+0000","last_active":"2024-09-14T04:02:51.340655+0000","last_peered":"2024-09-14T04:02:51.340655+0000","last_clean":"2024-09-14T04:02:51.340655+0000","last_became_active":"2024-09-14T04:01:59.051831+0000","last_became_peered":"2024-09-14T04:01:59.051831+0000","last_unstale":"2024-09-14T04:02:51.340655+0000","last_undegraded":"2024-09-14T04:02:51.340655+0000","last_fullsized":"2024-09-14T04:02:51.340655+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:15.224478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341485+0000","last_change":"2024-09-14T04:01:59.055460+0000","last_active":"2024-09-14T04:02:50.341485+0000","last_peered":"2024-09-14T04:02:50.341485+0000","last_clean":"2024-09-14T04:02:50.341485+0000","last_became_active":"2024-09-14T04:01:59.055064+0000","last_became_peered":"2024-09-14T04:01:59.055064+0000","last_unstale":"2024-09-14T04:02:50.341485+0000","last_undegraded":"2024-09-14T04:02:50.341485+0000","last_fullsized":"2024-09-14T04:02:50.341485+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:52.089107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340265+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:02:51.340265+0000","last_peered":"2024-09-14T04:02:51.340265+0000","last_clean":"2024-09-14T04:02:51.340265+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:02:51.340265+0000","last_undegraded":"2024-09-14T04:02:51.340265+0000","last_fullsized":"2024-09-14T04:02:51.340265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917955+0000","last_change":"2024-09-14T04:02:33.194317+0000","last_active":"2024-09-14T04:02:51.917955+0000","last_peered":"2024-09-14T04:02:51.917955+0000","last_clean":"2024-09-14T04:02:51.917955+0000","last_became_active":"2024-09-14T04:02:33.193981+0000","last_became_peered":"2024-09-14T04:02:33.193981+0000","last_unstale":"2024-09-14T04:02:51.917955+0000","last_undegraded":"2024-09-14T04:02:51.917955+0000","last_fullsized":"2024-09-14T04:02:51.917955+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:35:21.725699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343573+0000","last_change":"2024-09-14T04:02:33.192562+0000","last_active":"2024-09-14T04:02:51.343573+0000","last_peered":"2024-09-14T04:02:51.343573+0000","last_clean":"2024-09-14T04:02:51.343573+0000","last_became_active":"2024-09-14T04:02:33.192185+0000","last_became_peered":"2024-09-14T04:02:33.192185+0000","last_unstale":"2024-09-14T04:02:51.343573+0000","last_undegraded":"2024-09-14T04:02:51.343573+0000","last_fullsized":"2024-09-14T04:02:51.343573+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:34:52.201865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":429,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918623+0000","last_change":"2024-09-14T03:57:07.974148+0000","last_active":"2024-09-14T04:02:51.918623+0000","last_peered":"2024-09-14T04:02:51.918623+0000","last_clean":"2024-09-14T04:02:51.918623+0000","last_became_active":"2024-09-14T03:57:07.973612+0000","last_became_peered":"2024-09-14T03:57:07.973612+0000","last_unstale":"2024-09-14T04:02:51.918623+0000","last_undegraded":"2024-09-14T04:02:51.918623+0000","last_fullsized":"2024-09-14T04:02:51.918623+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:13.349866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338527+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:02:50.338527+0000","last_peered":"2024-09-14T04:02:50.338527+0000","last_clean":"2024-09-14T04:02:50.338527+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:02:50.338527+0000","last_undegraded":"2024-09-14T04:02:50.338527+0000","last_fullsized":"2024-09-14T04:02:50.338527+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918082+0000","last_change":"2024-09-14T04:02:33.189844+0000","last_active":"2024-09-14T04:02:51.918082+0000","last_peered":"2024-09-14T04:02:51.918082+0000","last_clean":"2024-09-14T04:02:51.918082+0000","last_became_active":"2024-09-14T04:02:33.189492+0000","last_became_peered":"2024-09-14T04:02:33.189492+0000","last_unstale":"2024-09-14T04:02:51.918082+0000","last_undegraded":"2024-09-14T04:02:51.918082+0000","last_fullsized":"2024-09-14T04:02:51.918082+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:28.814763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182854+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:02:33.182854+0000","last_peered":"2024-09-14T04:02:33.182854+0000","last_clean":"2024-09-14T04:02:33.182854+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:02:33.182854+0000","last_undegraded":"2024-09-14T04:02:33.182854+0000","last_fullsized":"2024-09-14T04:02:33.182854+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":126,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342266+0000","last_change":"2024-09-14T03:57:55.681501+0000","last_active":"2024-09-14T04:02:50.342266+0000","last_peered":"2024-09-14T04:02:50.342266+0000","last_clean":"2024-09-14T04:02:50.342266+0000","last_became_active":"2024-09-14T03:57:55.681104+0000","last_became_peered":"2024-09-14T03:57:55.681104+0000","last_unstale":"2024-09-14T04:02:50.342266+0000","last_undegraded":"2024-09-14T04:02:50.342266+0000","last_fullsized":"2024-09-14T04:02:50.342266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:54.441046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342257+0000","last_change":"2024-09-14T04:02:33.189629+0000","last_active":"2024-09-14T04:02:51.342257+0000","last_peered":"2024-09-14T04:02:51.342257+0000","last_clean":"2024-09-14T04:02:51.342257+0000","last_became_active":"2024-09-14T04:02:33.189317+0000","last_became_peered":"2024-09-14T04:02:33.189317+0000","last_unstale":"2024-09-14T04:02:51.342257+0000","last_undegraded":"2024-09-14T04:02:51.342257+0000","last_fullsized":"2024-09-14T04:02:51.342257+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:09.638142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342624+0000","last_change":"2024-09-14T04:01:59.060194+0000","last_active":"2024-09-14T04:02:51.342624+0000","last_peered":"2024-09-14T04:02:51.342624+0000","last_clean":"2024-09-14T04:02:51.342624+0000","last_became_active":"2024-09-14T04:01:59.059877+0000","last_became_peered":"2024-09-14T04:01:59.059877+0000","last_unstale":"2024-09-14T04:02:51.342624+0000","last_undegraded":"2024-09-14T04:02:51.342624+0000","last_fullsized":"2024-09-14T04:02:51.342624+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:34.061157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918185+0000","last_change":"2024-09-14T04:02:33.186950+0000","last_active":"2024-09-14T04:02:51.918185+0000","last_peered":"2024-09-14T04:02:51.918185+0000","last_clean":"2024-09-14T04:02:51.918185+0000","last_became_active":"2024-09-14T04:02:33.186615+0000","last_became_peered":"2024-09-14T04:02:33.186615+0000","last_unstale":"2024-09-14T04:02:51.918185+0000","last_undegraded":"2024-09-14T04:02:51.918185+0000","last_fullsized":"2024-09-14T04:02:51.918185+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:26.114627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342698+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:02:51.342698+0000","last_peered":"2024-09-14T04:02:51.342698+0000","last_clean":"2024-09-14T04:02:51.342698+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:02:51.342698+0000","last_undegraded":"2024-09-14T04:02:51.342698+0000","last_fullsized":"2024-09-14T04:02:51.342698+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343360+0000","last_change":"2024-09-14T03:57:55.641841+0000","last_active":"2024-09-14T04:02:51.343360+0000","last_peered":"2024-09-14T04:02:51.343360+0000","last_clean":"2024-09-14T04:02:51.343360+0000","last_became_active":"2024-09-14T03:57:55.638515+0000","last_became_peered":"2024-09-14T03:57:55.638515+0000","last_unstale":"2024-09-14T04:02:51.343360+0000","last_undegraded":"2024-09-14T04:02:51.343360+0000","last_fullsized":"2024-09-14T04:02:51.343360+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:07.687766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":113,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338437+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:02:50.338437+0000","last_peered":"2024-09-14T04:02:50.338437+0000","last_clean":"2024-09-14T04:02:50.338437+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:02:50.338437+0000","last_undegraded":"2024-09-14T04:02:50.338437+0000","last_fullsized":"2024-09-14T04:02:50.338437+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341728+0000","last_change":"2024-09-14T03:57:07.985126+0000","last_active":"2024-09-14T04:02:50.341728+0000","last_peered":"2024-09-14T04:02:50.341728+0000","last_clean":"2024-09-14T04:02:50.341728+0000","last_became_active":"2024-09-14T03:57:07.984427+0000","last_became_peered":"2024-09-14T03:57:07.984427+0000","last_unstale":"2024-09-14T04:02:50.341728+0000","last_undegraded":"2024-09-14T04:02:50.341728+0000","last_fullsized":"2024-09-14T04:02:50.341728+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:23.230268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338533+0000","last_change":"2024-09-14T04:02:33.189850+0000","last_active":"2024-09-14T04:02:50.338533+0000","last_peered":"2024-09-14T04:02:50.338533+0000","last_clean":"2024-09-14T04:02:50.338533+0000","last_became_active":"2024-09-14T04:02:33.189444+0000","last_became_peered":"2024-09-14T04:02:33.189444+0000","last_unstale":"2024-09-14T04:02:50.338533+0000","last_undegraded":"2024-09-14T04:02:50.338533+0000","last_fullsized":"2024-09-14T04:02:50.338533+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:23.120760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338905+0000","last_change":"2024-09-14T04:01:59.052874+0000","last_active":"2024-09-14T04:02:50.338905+0000","last_peered":"2024-09-14T04:02:50.338905+0000","last_clean":"2024-09-14T04:02:50.338905+0000","last_became_active":"2024-09-14T04:01:59.052455+0000","last_became_peered":"2024-09-14T04:01:59.052455+0000","last_unstale":"2024-09-14T04:02:50.338905+0000","last_undegraded":"2024-09-14T04:02:50.338905+0000","last_fullsized":"2024-09-14T04:02:50.338905+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:05.518917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343376+0000","last_change":"2024-09-14T03:57:55.656097+0000","last_active":"2024-09-14T04:02:51.343376+0000","last_peered":"2024-09-14T04:02:51.343376+0000","last_clean":"2024-09-14T04:02:51.343376+0000","last_became_active":"2024-09-14T03:57:55.649947+0000","last_became_peered":"2024-09-14T03:57:55.649947+0000","last_unstale":"2024-09-14T04:02:51.343376+0000","last_undegraded":"2024-09-14T04:02:51.343376+0000","last_fullsized":"2024-09-14T04:02:51.343376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.222621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342967+0000","last_change":"2024-09-14T03:57:55.667306+0000","last_active":"2024-09-14T04:02:51.342967+0000","last_peered":"2024-09-14T04:02:51.342967+0000","last_clean":"2024-09-14T04:02:51.342967+0000","last_became_active":"2024-09-14T03:57:55.661348+0000","last_became_peered":"2024-09-14T03:57:55.661348+0000","last_unstale":"2024-09-14T04:02:51.342967+0000","last_undegraded":"2024-09-14T04:02:51.342967+0000","last_fullsized":"2024-09-14T04:02:51.342967+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:25.408014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343274+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:02:51.343274+0000","last_peered":"2024-09-14T04:02:51.343274+0000","last_clean":"2024-09-14T04:02:51.343274+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:02:51.343274+0000","last_undegraded":"2024-09-14T04:02:51.343274+0000","last_fullsized":"2024-09-14T04:02:51.343274+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342983+0000","last_change":"2024-09-14T03:57:07.974119+0000","last_active":"2024-09-14T04:02:50.342983+0000","last_peered":"2024-09-14T04:02:50.342983+0000","last_clean":"2024-09-14T04:02:50.342983+0000","last_became_active":"2024-09-14T03:57:07.973617+0000","last_became_peered":"2024-09-14T03:57:07.973617+0000","last_unstale":"2024-09-14T04:02:50.342983+0000","last_undegraded":"2024-09-14T04:02:50.342983+0000","last_fullsized":"2024-09-14T04:02:50.342983+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:55.426883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342856+0000","last_change":"2024-09-14T03:57:07.984238+0000","last_active":"2024-09-14T04:02:51.342856+0000","last_peered":"2024-09-14T04:02:51.342856+0000","last_clean":"2024-09-14T04:02:51.342856+0000","last_became_active":"2024-09-14T03:57:07.983677+0000","last_became_peered":"2024-09-14T03:57:07.983677+0000","last_unstale":"2024-09-14T04:02:51.342856+0000","last_undegraded":"2024-09-14T04:02:51.342856+0000","last_fullsized":"2024-09-14T04:02:51.342856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:44:40.642082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918492+0000","last_change":"2024-09-14T04:02:33.190590+0000","last_active":"2024-09-14T04:02:51.918492+0000","last_peered":"2024-09-14T04:02:51.918492+0000","last_clean":"2024-09-14T04:02:51.918492+0000","last_became_active":"2024-09-14T04:02:33.190109+0000","last_became_peered":"2024-09-14T04:02:33.190109+0000","last_unstale":"2024-09-14T04:02:51.918492+0000","last_undegraded":"2024-09-14T04:02:51.918492+0000","last_fullsized":"2024-09-14T04:02:51.918492+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:53:10.464962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343631+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:02:51.343631+0000","last_peered":"2024-09-14T04:02:51.343631+0000","last_clean":"2024-09-14T04:02:51.343631+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:02:51.343631+0000","last_undegraded":"2024-09-14T04:02:51.343631+0000","last_fullsized":"2024-09-14T04:02:51.343631+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917900+0000","last_change":"2024-09-14T04:01:59.057227+0000","last_active":"2024-09-14T04:02:51.917900+0000","last_peered":"2024-09-14T04:02:51.917900+0000","last_clean":"2024-09-14T04:02:51.917900+0000","last_became_active":"2024-09-14T04:01:59.056799+0000","last_became_peered":"2024-09-14T04:01:59.056799+0000","last_unstale":"2024-09-14T04:02:51.917900+0000","last_undegraded":"2024-09-14T04:02:51.917900+0000","last_fullsized":"2024-09-14T04:02:51.917900+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:54.217497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":101,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342893+0000","last_change":"2024-09-14T03:57:55.649322+0000","last_active":"2024-09-14T04:02:51.342893+0000","last_peered":"2024-09-14T04:02:51.342893+0000","last_clean":"2024-09-14T04:02:51.342893+0000","last_became_active":"2024-09-14T03:57:55.644934+0000","last_became_peered":"2024-09-14T03:57:55.644934+0000","last_unstale":"2024-09-14T04:02:51.342893+0000","last_undegraded":"2024-09-14T04:02:51.342893+0000","last_fullsized":"2024-09-14T04:02:51.342893+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:35.277557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625457+0000","last_change":"2024-09-14T04:01:59.055291+0000","last_active":"2024-09-14T04:02:51.625457+0000","last_peered":"2024-09-14T04:02:51.625457+0000","last_clean":"2024-09-14T04:02:51.625457+0000","last_became_active":"2024-09-14T04:01:59.054919+0000","last_became_peered":"2024-09-14T04:01:59.054919+0000","last_unstale":"2024-09-14T04:02:51.625457+0000","last_undegraded":"2024-09-14T04:02:51.625457+0000","last_fullsized":"2024-09-14T04:02:51.625457+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:22.638368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341322+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:02:50.341322+0000","last_peered":"2024-09-14T04:02:50.341322+0000","last_clean":"2024-09-14T04:02:50.341322+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:02:50.341322+0000","last_undegraded":"2024-09-14T04:02:50.341322+0000","last_fullsized":"2024-09-14T04:02:50.341322+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341483+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:02:50.341483+0000","last_peered":"2024-09-14T04:02:50.341483+0000","last_clean":"2024-09-14T04:02:50.341483+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:02:50.341483+0000","last_undegraded":"2024-09-14T04:02:50.341483+0000","last_fullsized":"2024-09-14T04:02:50.341483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341694+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:02:50.341694+0000","last_peered":"2024-09-14T04:02:50.341694+0000","last_clean":"2024-09-14T04:02:50.341694+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:02:50.341694+0000","last_undegraded":"2024-09-14T04:02:50.341694+0000","last_fullsized":"2024-09-14T04:02:50.341694+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342812+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:02:51.342812+0000","last_peered":"2024-09-14T04:02:51.342812+0000","last_clean":"2024-09-14T04:02:51.342812+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:02:51.342812+0000","last_undegraded":"2024-09-14T04:02:51.342812+0000","last_fullsized":"2024-09-14T04:02:51.342812+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342777+0000","last_change":"2024-09-14T04:02:33.192294+0000","last_active":"2024-09-14T04:02:50.342777+0000","last_peered":"2024-09-14T04:02:50.342777+0000","last_clean":"2024-09-14T04:02:50.342777+0000","last_became_active":"2024-09-14T04:02:33.191886+0000","last_became_peered":"2024-09-14T04:02:33.191886+0000","last_unstale":"2024-09-14T04:02:50.342777+0000","last_undegraded":"2024-09-14T04:02:50.342777+0000","last_fullsized":"2024-09-14T04:02:50.342777+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:37:31.901824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338671+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:02:50.338671+0000","last_peered":"2024-09-14T04:02:50.338671+0000","last_clean":"2024-09-14T04:02:50.338671+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:02:50.338671+0000","last_undegraded":"2024-09-14T04:02:50.338671+0000","last_fullsized":"2024-09-14T04:02:50.338671+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918187+0000","last_change":"2024-09-14T04:02:33.187805+0000","last_active":"2024-09-14T04:02:51.918187+0000","last_peered":"2024-09-14T04:02:51.918187+0000","last_clean":"2024-09-14T04:02:51.918187+0000","last_became_active":"2024-09-14T04:02:33.187425+0000","last_became_peered":"2024-09-14T04:02:33.187425+0000","last_unstale":"2024-09-14T04:02:51.918187+0000","last_undegraded":"2024-09-14T04:02:51.918187+0000","last_fullsized":"2024-09-14T04:02:51.918187+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:32.471118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343272+0000","last_change":"2024-09-14T04:02:33.197914+0000","last_active":"2024-09-14T04:02:51.343272+0000","last_peered":"2024-09-14T04:02:51.343272+0000","last_clean":"2024-09-14T04:02:51.343272+0000","last_became_active":"2024-09-14T04:02:33.197447+0000","last_became_peered":"2024-09-14T04:02:33.197447+0000","last_unstale":"2024-09-14T04:02:51.343272+0000","last_undegraded":"2024-09-14T04:02:51.343272+0000","last_fullsized":"2024-09-14T04:02:51.343272+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:08:47.196932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918063+0000","last_change":"2024-09-14T04:02:33.196157+0000","last_active":"2024-09-14T04:02:51.918063+0000","last_peered":"2024-09-14T04:02:51.918063+0000","last_clean":"2024-09-14T04:02:51.918063+0000","last_became_active":"2024-09-14T04:02:33.195682+0000","last_became_peered":"2024-09-14T04:02:33.195682+0000","last_unstale":"2024-09-14T04:02:51.918063+0000","last_undegraded":"2024-09-14T04:02:51.918063+0000","last_fullsized":"2024-09-14T04:02:51.918063+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:04.357130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.183388+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:02:33.183388+0000","last_peered":"2024-09-14T04:02:33.183388+0000","last_clean":"2024-09-14T04:02:33.183388+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:02:33.183388+0000","last_undegraded":"2024-09-14T04:02:33.183388+0000","last_fullsized":"2024-09-14T04:02:33.183388+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918026+0000","last_change":"2024-09-14T04:02:33.195513+0000","last_active":"2024-09-14T04:02:51.918026+0000","last_peered":"2024-09-14T04:02:51.918026+0000","last_clean":"2024-09-14T04:02:51.918026+0000","last_became_active":"2024-09-14T04:02:33.195109+0000","last_became_peered":"2024-09-14T04:02:33.195109+0000","last_unstale":"2024-09-14T04:02:51.918026+0000","last_undegraded":"2024-09-14T04:02:51.918026+0000","last_fullsized":"2024-09-14T04:02:51.918026+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:30.591898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342747+0000","last_change":"2024-09-14T04:02:33.198166+0000","last_active":"2024-09-14T04:02:51.342747+0000","last_peered":"2024-09-14T04:02:51.342747+0000","last_clean":"2024-09-14T04:02:51.342747+0000","last_became_active":"2024-09-14T04:02:33.197287+0000","last_became_peered":"2024-09-14T04:02:33.197287+0000","last_unstale":"2024-09-14T04:02:51.342747+0000","last_undegraded":"2024-09-14T04:02:51.342747+0000","last_fullsized":"2024-09-14T04:02:51.342747+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:41:48.137435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342622+0000","last_change":"2024-09-14T04:02:33.192061+0000","last_active":"2024-09-14T04:02:51.342622+0000","last_peered":"2024-09-14T04:02:51.342622+0000","last_clean":"2024-09-14T04:02:51.342622+0000","last_became_active":"2024-09-14T04:02:33.191529+0000","last_became_peered":"2024-09-14T04:02:33.191529+0000","last_unstale":"2024-09-14T04:02:51.342622+0000","last_undegraded":"2024-09-14T04:02:51.342622+0000","last_fullsized":"2024-09-14T04:02:51.342622+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:06.031084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918787+0000","last_change":"2024-09-14T04:02:33.191433+0000","last_active":"2024-09-14T04:02:51.918787+0000","last_peered":"2024-09-14T04:02:51.918787+0000","last_clean":"2024-09-14T04:02:51.918787+0000","last_became_active":"2024-09-14T04:02:33.190554+0000","last_became_peered":"2024-09-14T04:02:33.190554+0000","last_unstale":"2024-09-14T04:02:51.918787+0000","last_undegraded":"2024-09-14T04:02:51.918787+0000","last_fullsized":"2024-09-14T04:02:51.918787+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:58.975784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342433+0000","last_change":"2024-09-14T04:01:59.053348+0000","last_active":"2024-09-14T04:02:51.342433+0000","last_peered":"2024-09-14T04:02:51.342433+0000","last_clean":"2024-09-14T04:02:51.342433+0000","last_became_active":"2024-09-14T04:01:59.053004+0000","last_became_peered":"2024-09-14T04:01:59.053004+0000","last_unstale":"2024-09-14T04:02:51.342433+0000","last_undegraded":"2024-09-14T04:02:51.342433+0000","last_fullsized":"2024-09-14T04:02:51.342433+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:56:28.268868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918249+0000","last_change":"2024-09-14T04:02:33.191341+0000","last_active":"2024-09-14T04:02:51.918249+0000","last_peered":"2024-09-14T04:02:51.918249+0000","last_clean":"2024-09-14T04:02:51.918249+0000","last_became_active":"2024-09-14T04:02:33.190081+0000","last_became_peered":"2024-09-14T04:02:33.190081+0000","last_unstale":"2024-09-14T04:02:51.918249+0000","last_undegraded":"2024-09-14T04:02:51.918249+0000","last_fullsized":"2024-09-14T04:02:51.918249+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:52.740363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342908+0000","last_change":"2024-09-14T04:01:59.057068+0000","last_active":"2024-09-14T04:02:51.342908+0000","last_peered":"2024-09-14T04:02:51.342908+0000","last_clean":"2024-09-14T04:02:51.342908+0000","last_became_active":"2024-09-14T04:01:59.056702+0000","last_became_peered":"2024-09-14T04:01:59.056702+0000","last_unstale":"2024-09-14T04:02:51.342908+0000","last_undegraded":"2024-09-14T04:02:51.342908+0000","last_fullsized":"2024-09-14T04:02:51.342908+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:23.708120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341445+0000","last_change":"2024-09-14T04:02:33.192285+0000","last_active":"2024-09-14T04:02:50.341445+0000","last_peered":"2024-09-14T04:02:50.341445+0000","last_clean":"2024-09-14T04:02:50.341445+0000","last_became_active":"2024-09-14T04:02:33.191832+0000","last_became_peered":"2024-09-14T04:02:33.191832+0000","last_unstale":"2024-09-14T04:02:50.341445+0000","last_undegraded":"2024-09-14T04:02:50.341445+0000","last_fullsized":"2024-09-14T04:02:50.341445+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:38:16.739279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":136,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.346840+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:02:51.346840+0000","last_peered":"2024-09-14T04:02:51.346840+0000","last_clean":"2024-09-14T04:02:51.346840+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:02:51.346840+0000","last_undegraded":"2024-09-14T04:02:51.346840+0000","last_fullsized":"2024-09-14T04:02:51.346840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341322+0000","last_change":"2024-09-14T04:02:33.194384+0000","last_active":"2024-09-14T04:02:50.341322+0000","last_peered":"2024-09-14T04:02:50.341322+0000","last_clean":"2024-09-14T04:02:50.341322+0000","last_became_active":"2024-09-14T04:02:33.194027+0000","last_became_peered":"2024-09-14T04:02:33.194027+0000","last_unstale":"2024-09-14T04:02:50.341322+0000","last_undegraded":"2024-09-14T04:02:50.341322+0000","last_fullsized":"2024-09-14T04:02:50.341322+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:46:29.339591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338808+0000","last_change":"2024-09-14T03:57:08.990606+0000","last_active":"2024-09-14T04:02:50.338808+0000","last_peered":"2024-09-14T04:02:50.338808+0000","last_clean":"2024-09-14T04:02:50.338808+0000","last_became_active":"2024-09-14T03:57:08.979113+0000","last_became_peered":"2024-09-14T03:57:08.979113+0000","last_unstale":"2024-09-14T04:02:50.338808+0000","last_undegraded":"2024-09-14T04:02:50.338808+0000","last_fullsized":"2024-09-14T04:02:50.338808+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:53.542287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918089+0000","last_change":"2024-09-14T04:02:33.194588+0000","last_active":"2024-09-14T04:02:51.918089+0000","last_peered":"2024-09-14T04:02:51.918089+0000","last_clean":"2024-09-14T04:02:51.918089+0000","last_became_active":"2024-09-14T04:02:33.193732+0000","last_became_peered":"2024-09-14T04:02:33.193732+0000","last_unstale":"2024-09-14T04:02:51.918089+0000","last_undegraded":"2024-09-14T04:02:51.918089+0000","last_fullsized":"2024-09-14T04:02:51.918089+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:50.529572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918743+0000","last_change":"2024-09-14T04:02:33.193693+0000","last_active":"2024-09-14T04:02:51.918743+0000","last_peered":"2024-09-14T04:02:51.918743+0000","last_clean":"2024-09-14T04:02:51.918743+0000","last_became_active":"2024-09-14T04:02:33.192884+0000","last_became_peered":"2024-09-14T04:02:33.192884+0000","last_unstale":"2024-09-14T04:02:51.918743+0000","last_undegraded":"2024-09-14T04:02:51.918743+0000","last_fullsized":"2024-09-14T04:02:51.918743+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:21.242232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341118+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:02:50.341118+0000","last_peered":"2024-09-14T04:02:50.341118+0000","last_clean":"2024-09-14T04:02:50.341118+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:02:50.341118+0000","last_undegraded":"2024-09-14T04:02:50.341118+0000","last_fullsized":"2024-09-14T04:02:50.341118+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":102,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343134+0000","last_change":"2024-09-14T04:01:59.054141+0000","last_active":"2024-09-14T04:02:51.343134+0000","last_peered":"2024-09-14T04:02:51.343134+0000","last_clean":"2024-09-14T04:02:51.343134+0000","last_became_active":"2024-09-14T04:01:59.053397+0000","last_became_peered":"2024-09-14T04:01:59.053397+0000","last_unstale":"2024-09-14T04:02:51.343134+0000","last_undegraded":"2024-09-14T04:02:51.343134+0000","last_fullsized":"2024-09-14T04:02:51.343134+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:38.323001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13714,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201553,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466452,"kb_used_data":2184,"kb_used_omap":1,"kb_used_meta":464254,"kb_avail":93278700,"statfs":{"total":95995035648,"available":95517388800,"internally_reserved":0,"allocated":2236416,"data_stored":1440220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":475396555},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463191,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416392,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":93328760,"statfs":{"total":95995035648,"available":95568650240,"internally_reserved":0,"allocated":2224128,"data_stored":1439573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":424147399},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724827,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640580,"kb_used_data":1588,"kb_used_omap":1,"kb_used_meta":638974,"kb_avail":93104572,"statfs":{"total":95995035648,"available":95339081728,"internally_reserved":0,"allocated":1626112,"data_stored":845304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":654309830},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986463,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560208,"kb_used_data":1600,"kb_used_omap":1,"kb_used_meta":558590,"kb_avail":93184944,"statfs":{"total":95995035648,"available":95421382656,"internally_reserved":0,"allocated":1638400,"data_stored":849543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":571996614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748492,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84988,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":83387,"kb_avail":93660164,"statfs":{"total":95995035648,"available":95908007936,"internally_reserved":0,"allocated":1622016,"data_stored":845016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":85388844},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912019,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76036,"kb_used_data":1588,"kb_used_omap":6,"kb_used_meta":74425,"kb_avail":93669116,"statfs":{"total":95995035648,"available":95917174784,"internally_reserved":0,"allocated":1626112,"data_stored":845542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":76211769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468036,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190392,"kb_used_data":1580,"kb_used_omap":5,"kb_used_meta":188794,"kb_avail":93554760,"statfs":{"total":95995035648,"available":95800074240,"internally_reserved":0,"allocated":1617920,"data_stored":843525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":193325407},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":102,"seq":438086664269,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156872,"kb_used_data":2172,"kb_used_omap":5,"kb_used_meta":154682,"kb_avail":93588280,"statfs":{"total":95995035648,"available":95834398720,"internally_reserved":0,"allocated":2224128,"data_stored":1435932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5976,"internal_metadata":158394536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4382,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:03:26.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:03:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:26 smithi012 ceph-mon[101180]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:26.805 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:26.926 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:26] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.51.0" 2024-09-14T04:03:27.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:27 smithi001 ceph-mon[107399]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:27.714 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:27 smithi001 ceph-mon[112165]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:27 smithi012 ceph-mon[101180]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:28.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:28 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:28 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:28.730 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:03:28.731 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:03:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:28 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:29.437 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":429,"stamp":"2024-09-14T04:03:27.284010+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771138,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2591920,"kb_used_data":14468,"kb_used_omap":28,"kb_used_meta":2577315,"kb_avail":747369296,"statfs":{"total":767960285184,"available":765306159104,"internally_reserved":0,"allocated":14815232,"data_stored":8544655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29302,"internal_metadata":2639170954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002970"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918602+0000","last_change":"2024-09-14T04:02:33.186418+0000","last_active":"2024-09-14T04:02:51.918602+0000","last_peered":"2024-09-14T04:02:51.918602+0000","last_clean":"2024-09-14T04:02:51.918602+0000","last_became_active":"2024-09-14T04:02:33.186052+0000","last_became_peered":"2024-09-14T04:02:33.186052+0000","last_unstale":"2024-09-14T04:02:51.918602+0000","last_undegraded":"2024-09-14T04:02:51.918602+0000","last_fullsized":"2024-09-14T04:02:51.918602+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:04.240828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":89,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342633+0000","last_change":"2024-09-14T04:01:59.055661+0000","last_active":"2024-09-14T04:02:51.342633+0000","last_peered":"2024-09-14T04:02:51.342633+0000","last_clean":"2024-09-14T04:02:51.342633+0000","last_became_active":"2024-09-14T04:01:59.055301+0000","last_became_peered":"2024-09-14T04:01:59.055301+0000","last_unstale":"2024-09-14T04:02:51.342633+0000","last_undegraded":"2024-09-14T04:02:51.342633+0000","last_fullsized":"2024-09-14T04:02:51.342633+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:34:27.575772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338958+0000","last_change":"2024-09-14T04:02:33.189718+0000","last_active":"2024-09-14T04:02:50.338958+0000","last_peered":"2024-09-14T04:02:50.338958+0000","last_clean":"2024-09-14T04:02:50.338958+0000","last_became_active":"2024-09-14T04:02:33.189342+0000","last_became_peered":"2024-09-14T04:02:33.189342+0000","last_unstale":"2024-09-14T04:02:50.338958+0000","last_undegraded":"2024-09-14T04:02:50.338958+0000","last_fullsized":"2024-09-14T04:02:50.338958+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:51:37.536543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342588+0000","last_change":"2024-09-14T04:02:33.196773+0000","last_active":"2024-09-14T04:02:51.342588+0000","last_peered":"2024-09-14T04:02:51.342588+0000","last_clean":"2024-09-14T04:02:51.342588+0000","last_became_active":"2024-09-14T04:02:33.196447+0000","last_became_peered":"2024-09-14T04:02:33.196447+0000","last_unstale":"2024-09-14T04:02:51.342588+0000","last_undegraded":"2024-09-14T04:02:51.342588+0000","last_fullsized":"2024-09-14T04:02:51.342588+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:51.157998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342796+0000","last_change":"2024-09-14T04:02:33.198492+0000","last_active":"2024-09-14T04:02:51.342796+0000","last_peered":"2024-09-14T04:02:51.342796+0000","last_clean":"2024-09-14T04:02:51.342796+0000","last_became_active":"2024-09-14T04:02:33.197804+0000","last_became_peered":"2024-09-14T04:02:33.197804+0000","last_unstale":"2024-09-14T04:02:51.342796+0000","last_undegraded":"2024-09-14T04:02:51.342796+0000","last_fullsized":"2024-09-14T04:02:51.342796+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:37.842086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342325+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:02:50.342325+0000","last_peered":"2024-09-14T04:02:50.342325+0000","last_clean":"2024-09-14T04:02:50.342325+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:02:50.342325+0000","last_undegraded":"2024-09-14T04:02:50.342325+0000","last_fullsized":"2024-09-14T04:02:50.342325+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343001+0000","last_change":"2024-09-14T03:57:55.641587+0000","last_active":"2024-09-14T04:02:51.343001+0000","last_peered":"2024-09-14T04:02:51.343001+0000","last_clean":"2024-09-14T04:02:51.343001+0000","last_became_active":"2024-09-14T03:57:55.640967+0000","last_became_peered":"2024-09-14T03:57:55.640967+0000","last_unstale":"2024-09-14T04:02:51.343001+0000","last_undegraded":"2024-09-14T04:02:51.343001+0000","last_fullsized":"2024-09-14T04:02:51.343001+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:05.024573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":103,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.181388+0000","last_change":"2024-09-14T03:57:07.969661+0000","last_active":"2024-09-14T04:02:33.181388+0000","last_peered":"2024-09-14T04:02:33.181388+0000","last_clean":"2024-09-14T04:02:33.181388+0000","last_became_active":"2024-09-14T03:57:07.969236+0000","last_became_peered":"2024-09-14T03:57:07.969236+0000","last_unstale":"2024-09-14T04:02:33.181388+0000","last_undegraded":"2024-09-14T04:02:33.181388+0000","last_fullsized":"2024-09-14T04:02:33.181388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:28.685717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.919228+0000","last_change":"2024-09-14T03:57:07.978483+0000","last_active":"2024-09-14T04:02:51.919228+0000","last_peered":"2024-09-14T04:02:51.919228+0000","last_clean":"2024-09-14T04:02:51.919228+0000","last_became_active":"2024-09-14T03:57:07.978076+0000","last_became_peered":"2024-09-14T03:57:07.978076+0000","last_unstale":"2024-09-14T04:02:51.919228+0000","last_undegraded":"2024-09-14T04:02:51.919228+0000","last_fullsized":"2024-09-14T04:02:51.919228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:33.126366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338345+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T04:02:50.338345+0000","last_peered":"2024-09-14T04:02:50.338345+0000","last_clean":"2024-09-14T04:02:50.338345+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T04:02:50.338345+0000","last_undegraded":"2024-09-14T04:02:50.338345+0000","last_fullsized":"2024-09-14T04:02:50.338345+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918439+0000","last_change":"2024-09-14T03:57:55.658263+0000","last_active":"2024-09-14T04:02:51.918439+0000","last_peered":"2024-09-14T04:02:51.918439+0000","last_clean":"2024-09-14T04:02:51.918439+0000","last_became_active":"2024-09-14T03:57:55.657817+0000","last_became_peered":"2024-09-14T03:57:55.657817+0000","last_unstale":"2024-09-14T04:02:51.918439+0000","last_undegraded":"2024-09-14T04:02:51.918439+0000","last_fullsized":"2024-09-14T04:02:51.918439+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:36.196909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341202+0000","last_change":"2024-09-14T04:02:33.196376+0000","last_active":"2024-09-14T04:02:50.341202+0000","last_peered":"2024-09-14T04:02:50.341202+0000","last_clean":"2024-09-14T04:02:50.341202+0000","last_became_active":"2024-09-14T04:02:33.195804+0000","last_became_peered":"2024-09-14T04:02:33.195804+0000","last_unstale":"2024-09-14T04:02:50.341202+0000","last_undegraded":"2024-09-14T04:02:50.341202+0000","last_fullsized":"2024-09-14T04:02:50.341202+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:58:02.150113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918423+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T04:02:51.918423+0000","last_peered":"2024-09-14T04:02:51.918423+0000","last_clean":"2024-09-14T04:02:51.918423+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T04:02:51.918423+0000","last_undegraded":"2024-09-14T04:02:51.918423+0000","last_fullsized":"2024-09-14T04:02:51.918423+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182266+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:02:33.182266+0000","last_peered":"2024-09-14T04:02:33.182266+0000","last_clean":"2024-09-14T04:02:33.182266+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:02:33.182266+0000","last_undegraded":"2024-09-14T04:02:33.182266+0000","last_fullsized":"2024-09-14T04:02:33.182266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.598290+0000","last_change":"2024-09-14T04:02:33.189804+0000","last_active":"2024-09-14T04:02:50.598290+0000","last_peered":"2024-09-14T04:02:50.598290+0000","last_clean":"2024-09-14T04:02:50.598290+0000","last_became_active":"2024-09-14T04:02:33.189427+0000","last_became_peered":"2024-09-14T04:02:33.189427+0000","last_unstale":"2024-09-14T04:02:50.598290+0000","last_undegraded":"2024-09-14T04:02:50.598290+0000","last_fullsized":"2024-09-14T04:02:50.598290+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:35.027998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342724+0000","last_change":"2024-09-14T03:57:55.640069+0000","last_active":"2024-09-14T04:02:51.342724+0000","last_peered":"2024-09-14T04:02:51.342724+0000","last_clean":"2024-09-14T04:02:51.342724+0000","last_became_active":"2024-09-14T03:57:55.639653+0000","last_became_peered":"2024-09-14T03:57:55.639653+0000","last_unstale":"2024-09-14T04:02:51.342724+0000","last_undegraded":"2024-09-14T04:02:51.342724+0000","last_fullsized":"2024-09-14T04:02:51.342724+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:16.579752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918366+0000","last_change":"2024-09-14T04:02:33.189648+0000","last_active":"2024-09-14T04:02:51.918366+0000","last_peered":"2024-09-14T04:02:51.918366+0000","last_clean":"2024-09-14T04:02:51.918366+0000","last_became_active":"2024-09-14T04:02:33.188956+0000","last_became_peered":"2024-09-14T04:02:33.188956+0000","last_unstale":"2024-09-14T04:02:51.918366+0000","last_undegraded":"2024-09-14T04:02:51.918366+0000","last_fullsized":"2024-09-14T04:02:51.918366+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:14.851661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.384607+0000","last_change":"2024-09-14T04:02:33.194407+0000","last_active":"2024-09-14T04:02:50.384607+0000","last_peered":"2024-09-14T04:02:50.384607+0000","last_clean":"2024-09-14T04:02:50.384607+0000","last_became_active":"2024-09-14T04:02:33.194049+0000","last_became_peered":"2024-09-14T04:02:33.194049+0000","last_unstale":"2024-09-14T04:02:50.384607+0000","last_undegraded":"2024-09-14T04:02:50.384607+0000","last_fullsized":"2024-09-14T04:02:50.384607+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:09.389524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182415+0000","last_change":"2024-09-14T03:57:55.680448+0000","last_active":"2024-09-14T04:02:33.182415+0000","last_peered":"2024-09-14T04:02:33.182415+0000","last_clean":"2024-09-14T04:02:33.182415+0000","last_became_active":"2024-09-14T03:57:55.680199+0000","last_became_peered":"2024-09-14T03:57:55.680199+0000","last_unstale":"2024-09-14T04:02:33.182415+0000","last_undegraded":"2024-09-14T04:02:33.182415+0000","last_fullsized":"2024-09-14T04:02:33.182415+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:03.237323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340655+0000","last_change":"2024-09-14T04:01:59.052174+0000","last_active":"2024-09-14T04:02:51.340655+0000","last_peered":"2024-09-14T04:02:51.340655+0000","last_clean":"2024-09-14T04:02:51.340655+0000","last_became_active":"2024-09-14T04:01:59.051831+0000","last_became_peered":"2024-09-14T04:01:59.051831+0000","last_unstale":"2024-09-14T04:02:51.340655+0000","last_undegraded":"2024-09-14T04:02:51.340655+0000","last_fullsized":"2024-09-14T04:02:51.340655+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:15.224478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341485+0000","last_change":"2024-09-14T04:01:59.055460+0000","last_active":"2024-09-14T04:02:50.341485+0000","last_peered":"2024-09-14T04:02:50.341485+0000","last_clean":"2024-09-14T04:02:50.341485+0000","last_became_active":"2024-09-14T04:01:59.055064+0000","last_became_peered":"2024-09-14T04:01:59.055064+0000","last_unstale":"2024-09-14T04:02:50.341485+0000","last_undegraded":"2024-09-14T04:02:50.341485+0000","last_fullsized":"2024-09-14T04:02:50.341485+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:52.089107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340265+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:02:51.340265+0000","last_peered":"2024-09-14T04:02:51.340265+0000","last_clean":"2024-09-14T04:02:51.340265+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:02:51.340265+0000","last_undegraded":"2024-09-14T04:02:51.340265+0000","last_fullsized":"2024-09-14T04:02:51.340265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917955+0000","last_change":"2024-09-14T04:02:33.194317+0000","last_active":"2024-09-14T04:02:51.917955+0000","last_peered":"2024-09-14T04:02:51.917955+0000","last_clean":"2024-09-14T04:02:51.917955+0000","last_became_active":"2024-09-14T04:02:33.193981+0000","last_became_peered":"2024-09-14T04:02:33.193981+0000","last_unstale":"2024-09-14T04:02:51.917955+0000","last_undegraded":"2024-09-14T04:02:51.917955+0000","last_fullsized":"2024-09-14T04:02:51.917955+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:35:21.725699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343573+0000","last_change":"2024-09-14T04:02:33.192562+0000","last_active":"2024-09-14T04:02:51.343573+0000","last_peered":"2024-09-14T04:02:51.343573+0000","last_clean":"2024-09-14T04:02:51.343573+0000","last_became_active":"2024-09-14T04:02:33.192185+0000","last_became_peered":"2024-09-14T04:02:33.192185+0000","last_unstale":"2024-09-14T04:02:51.343573+0000","last_undegraded":"2024-09-14T04:02:51.343573+0000","last_fullsized":"2024-09-14T04:02:51.343573+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:34:52.201865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":429,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918623+0000","last_change":"2024-09-14T03:57:07.974148+0000","last_active":"2024-09-14T04:02:51.918623+0000","last_peered":"2024-09-14T04:02:51.918623+0000","last_clean":"2024-09-14T04:02:51.918623+0000","last_became_active":"2024-09-14T03:57:07.973612+0000","last_became_peered":"2024-09-14T03:57:07.973612+0000","last_unstale":"2024-09-14T04:02:51.918623+0000","last_undegraded":"2024-09-14T04:02:51.918623+0000","last_fullsized":"2024-09-14T04:02:51.918623+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:13.349866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338527+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:02:50.338527+0000","last_peered":"2024-09-14T04:02:50.338527+0000","last_clean":"2024-09-14T04:02:50.338527+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:02:50.338527+0000","last_undegraded":"2024-09-14T04:02:50.338527+0000","last_fullsized":"2024-09-14T04:02:50.338527+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918082+0000","last_change":"2024-09-14T04:02:33.189844+0000","last_active":"2024-09-14T04:02:51.918082+0000","last_peered":"2024-09-14T04:02:51.918082+0000","last_clean":"2024-09-14T04:02:51.918082+0000","last_became_active":"2024-09-14T04:02:33.189492+0000","last_became_peered":"2024-09-14T04:02:33.189492+0000","last_unstale":"2024-09-14T04:02:51.918082+0000","last_undegraded":"2024-09-14T04:02:51.918082+0000","last_fullsized":"2024-09-14T04:02:51.918082+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:28.814763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182854+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:02:33.182854+0000","last_peered":"2024-09-14T04:02:33.182854+0000","last_clean":"2024-09-14T04:02:33.182854+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:02:33.182854+0000","last_undegraded":"2024-09-14T04:02:33.182854+0000","last_fullsized":"2024-09-14T04:02:33.182854+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":126,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342266+0000","last_change":"2024-09-14T03:57:55.681501+0000","last_active":"2024-09-14T04:02:50.342266+0000","last_peered":"2024-09-14T04:02:50.342266+0000","last_clean":"2024-09-14T04:02:50.342266+0000","last_became_active":"2024-09-14T03:57:55.681104+0000","last_became_peered":"2024-09-14T03:57:55.681104+0000","last_unstale":"2024-09-14T04:02:50.342266+0000","last_undegraded":"2024-09-14T04:02:50.342266+0000","last_fullsized":"2024-09-14T04:02:50.342266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:54.441046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342257+0000","last_change":"2024-09-14T04:02:33.189629+0000","last_active":"2024-09-14T04:02:51.342257+0000","last_peered":"2024-09-14T04:02:51.342257+0000","last_clean":"2024-09-14T04:02:51.342257+0000","last_became_active":"2024-09-14T04:02:33.189317+0000","last_became_peered":"2024-09-14T04:02:33.189317+0000","last_unstale":"2024-09-14T04:02:51.342257+0000","last_undegraded":"2024-09-14T04:02:51.342257+0000","last_fullsized":"2024-09-14T04:02:51.342257+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:09.638142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342624+0000","last_change":"2024-09-14T04:01:59.060194+0000","last_active":"2024-09-14T04:02:51.342624+0000","last_peered":"2024-09-14T04:02:51.342624+0000","last_clean":"2024-09-14T04:02:51.342624+0000","last_became_active":"2024-09-14T04:01:59.059877+0000","last_became_peered":"2024-09-14T04:01:59.059877+0000","last_unstale":"2024-09-14T04:02:51.342624+0000","last_undegraded":"2024-09-14T04:02:51.342624+0000","last_fullsized":"2024-09-14T04:02:51.342624+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:34.061157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918185+0000","last_change":"2024-09-14T04:02:33.186950+0000","last_active":"2024-09-14T04:02:51.918185+0000","last_peered":"2024-09-14T04:02:51.918185+0000","last_clean":"2024-09-14T04:02:51.918185+0000","last_became_active":"2024-09-14T04:02:33.186615+0000","last_became_peered":"2024-09-14T04:02:33.186615+0000","last_unstale":"2024-09-14T04:02:51.918185+0000","last_undegraded":"2024-09-14T04:02:51.918185+0000","last_fullsized":"2024-09-14T04:02:51.918185+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:26.114627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342698+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:02:51.342698+0000","last_peered":"2024-09-14T04:02:51.342698+0000","last_clean":"2024-09-14T04:02:51.342698+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:02:51.342698+0000","last_undegraded":"2024-09-14T04:02:51.342698+0000","last_fullsized":"2024-09-14T04:02:51.342698+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343360+0000","last_change":"2024-09-14T03:57:55.641841+0000","last_active":"2024-09-14T04:02:51.343360+0000","last_peered":"2024-09-14T04:02:51.343360+0000","last_clean":"2024-09-14T04:02:51.343360+0000","last_became_active":"2024-09-14T03:57:55.638515+0000","last_became_peered":"2024-09-14T03:57:55.638515+0000","last_unstale":"2024-09-14T04:02:51.343360+0000","last_undegraded":"2024-09-14T04:02:51.343360+0000","last_fullsized":"2024-09-14T04:02:51.343360+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:07.687766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":113,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338437+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:02:50.338437+0000","last_peered":"2024-09-14T04:02:50.338437+0000","last_clean":"2024-09-14T04:02:50.338437+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:02:50.338437+0000","last_undegraded":"2024-09-14T04:02:50.338437+0000","last_fullsized":"2024-09-14T04:02:50.338437+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341728+0000","last_change":"2024-09-14T03:57:07.985126+0000","last_active":"2024-09-14T04:02:50.341728+0000","last_peered":"2024-09-14T04:02:50.341728+0000","last_clean":"2024-09-14T04:02:50.341728+0000","last_became_active":"2024-09-14T03:57:07.984427+0000","last_became_peered":"2024-09-14T03:57:07.984427+0000","last_unstale":"2024-09-14T04:02:50.341728+0000","last_undegraded":"2024-09-14T04:02:50.341728+0000","last_fullsized":"2024-09-14T04:02:50.341728+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:23.230268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338533+0000","last_change":"2024-09-14T04:02:33.189850+0000","last_active":"2024-09-14T04:02:50.338533+0000","last_peered":"2024-09-14T04:02:50.338533+0000","last_clean":"2024-09-14T04:02:50.338533+0000","last_became_active":"2024-09-14T04:02:33.189444+0000","last_became_peered":"2024-09-14T04:02:33.189444+0000","last_unstale":"2024-09-14T04:02:50.338533+0000","last_undegraded":"2024-09-14T04:02:50.338533+0000","last_fullsized":"2024-09-14T04:02:50.338533+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:23.120760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338905+0000","last_change":"2024-09-14T04:01:59.052874+0000","last_active":"2024-09-14T04:02:50.338905+0000","last_peered":"2024-09-14T04:02:50.338905+0000","last_clean":"2024-09-14T04:02:50.338905+0000","last_became_active":"2024-09-14T04:01:59.052455+0000","last_became_peered":"2024-09-14T04:01:59.052455+0000","last_unstale":"2024-09-14T04:02:50.338905+0000","last_undegraded":"2024-09-14T04:02:50.338905+0000","last_fullsized":"2024-09-14T04:02:50.338905+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:05.518917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343376+0000","last_change":"2024-09-14T03:57:55.656097+0000","last_active":"2024-09-14T04:02:51.343376+0000","last_peered":"2024-09-14T04:02:51.343376+0000","last_clean":"2024-09-14T04:02:51.343376+0000","last_became_active":"2024-09-14T03:57:55.649947+0000","last_became_peered":"2024-09-14T03:57:55.649947+0000","last_unstale":"2024-09-14T04:02:51.343376+0000","last_undegraded":"2024-09-14T04:02:51.343376+0000","last_fullsized":"2024-09-14T04:02:51.343376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.222621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342967+0000","last_change":"2024-09-14T03:57:55.667306+0000","last_active":"2024-09-14T04:02:51.342967+0000","last_peered":"2024-09-14T04:02:51.342967+0000","last_clean":"2024-09-14T04:02:51.342967+0000","last_became_active":"2024-09-14T03:57:55.661348+0000","last_became_peered":"2024-09-14T03:57:55.661348+0000","last_unstale":"2024-09-14T04:02:51.342967+0000","last_undegraded":"2024-09-14T04:02:51.342967+0000","last_fullsized":"2024-09-14T04:02:51.342967+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:25.408014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343274+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:02:51.343274+0000","last_peered":"2024-09-14T04:02:51.343274+0000","last_clean":"2024-09-14T04:02:51.343274+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:02:51.343274+0000","last_undegraded":"2024-09-14T04:02:51.343274+0000","last_fullsized":"2024-09-14T04:02:51.343274+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342983+0000","last_change":"2024-09-14T03:57:07.974119+0000","last_active":"2024-09-14T04:02:50.342983+0000","last_peered":"2024-09-14T04:02:50.342983+0000","last_clean":"2024-09-14T04:02:50.342983+0000","last_became_active":"2024-09-14T03:57:07.973617+0000","last_became_peered":"2024-09-14T03:57:07.973617+0000","last_unstale":"2024-09-14T04:02:50.342983+0000","last_undegraded":"2024-09-14T04:02:50.342983+0000","last_fullsized":"2024-09-14T04:02:50.342983+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:55.426883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342856+0000","last_change":"2024-09-14T03:57:07.984238+0000","last_active":"2024-09-14T04:02:51.342856+0000","last_peered":"2024-09-14T04:02:51.342856+0000","last_clean":"2024-09-14T04:02:51.342856+0000","last_became_active":"2024-09-14T03:57:07.983677+0000","last_became_peered":"2024-09-14T03:57:07.983677+0000","last_unstale":"2024-09-14T04:02:51.342856+0000","last_undegraded":"2024-09-14T04:02:51.342856+0000","last_fullsized":"2024-09-14T04:02:51.342856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:44:40.642082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918492+0000","last_change":"2024-09-14T04:02:33.190590+0000","last_active":"2024-09-14T04:02:51.918492+0000","last_peered":"2024-09-14T04:02:51.918492+0000","last_clean":"2024-09-14T04:02:51.918492+0000","last_became_active":"2024-09-14T04:02:33.190109+0000","last_became_peered":"2024-09-14T04:02:33.190109+0000","last_unstale":"2024-09-14T04:02:51.918492+0000","last_undegraded":"2024-09-14T04:02:51.918492+0000","last_fullsized":"2024-09-14T04:02:51.918492+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:53:10.464962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343631+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:02:51.343631+0000","last_peered":"2024-09-14T04:02:51.343631+0000","last_clean":"2024-09-14T04:02:51.343631+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:02:51.343631+0000","last_undegraded":"2024-09-14T04:02:51.343631+0000","last_fullsized":"2024-09-14T04:02:51.343631+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917900+0000","last_change":"2024-09-14T04:01:59.057227+0000","last_active":"2024-09-14T04:02:51.917900+0000","last_peered":"2024-09-14T04:02:51.917900+0000","last_clean":"2024-09-14T04:02:51.917900+0000","last_became_active":"2024-09-14T04:01:59.056799+0000","last_became_peered":"2024-09-14T04:01:59.056799+0000","last_unstale":"2024-09-14T04:02:51.917900+0000","last_undegraded":"2024-09-14T04:02:51.917900+0000","last_fullsized":"2024-09-14T04:02:51.917900+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:54.217497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":101,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342893+0000","last_change":"2024-09-14T03:57:55.649322+0000","last_active":"2024-09-14T04:02:51.342893+0000","last_peered":"2024-09-14T04:02:51.342893+0000","last_clean":"2024-09-14T04:02:51.342893+0000","last_became_active":"2024-09-14T03:57:55.644934+0000","last_became_peered":"2024-09-14T03:57:55.644934+0000","last_unstale":"2024-09-14T04:02:51.342893+0000","last_undegraded":"2024-09-14T04:02:51.342893+0000","last_fullsized":"2024-09-14T04:02:51.342893+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:35.277557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625457+0000","last_change":"2024-09-14T04:01:59.055291+0000","last_active":"2024-09-14T04:02:51.625457+0000","last_peered":"2024-09-14T04:02:51.625457+0000","last_clean":"2024-09-14T04:02:51.625457+0000","last_became_active":"2024-09-14T04:01:59.054919+0000","last_became_peered":"2024-09-14T04:01:59.054919+0000","last_unstale":"2024-09-14T04:02:51.625457+0000","last_undegraded":"2024-09-14T04:02:51.625457+0000","last_fullsized":"2024-09-14T04:02:51.625457+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:22.638368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340137+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:02:51.340137+0000","last_peered":"2024-09-14T04:02:51.340137+0000","last_clean":"2024-09-14T04:02:51.340137+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:02:51.340137+0000","last_undegraded":"2024-09-14T04:02:51.340137+0000","last_fullsized":"2024-09-14T04:02:51.340137+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341483+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:02:50.341483+0000","last_peered":"2024-09-14T04:02:50.341483+0000","last_clean":"2024-09-14T04:02:50.341483+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:02:50.341483+0000","last_undegraded":"2024-09-14T04:02:50.341483+0000","last_fullsized":"2024-09-14T04:02:50.341483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341694+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:02:50.341694+0000","last_peered":"2024-09-14T04:02:50.341694+0000","last_clean":"2024-09-14T04:02:50.341694+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:02:50.341694+0000","last_undegraded":"2024-09-14T04:02:50.341694+0000","last_fullsized":"2024-09-14T04:02:50.341694+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342812+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:02:51.342812+0000","last_peered":"2024-09-14T04:02:51.342812+0000","last_clean":"2024-09-14T04:02:51.342812+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:02:51.342812+0000","last_undegraded":"2024-09-14T04:02:51.342812+0000","last_fullsized":"2024-09-14T04:02:51.342812+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342777+0000","last_change":"2024-09-14T04:02:33.192294+0000","last_active":"2024-09-14T04:02:50.342777+0000","last_peered":"2024-09-14T04:02:50.342777+0000","last_clean":"2024-09-14T04:02:50.342777+0000","last_became_active":"2024-09-14T04:02:33.191886+0000","last_became_peered":"2024-09-14T04:02:33.191886+0000","last_unstale":"2024-09-14T04:02:50.342777+0000","last_undegraded":"2024-09-14T04:02:50.342777+0000","last_fullsized":"2024-09-14T04:02:50.342777+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:37:31.901824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338671+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:02:50.338671+0000","last_peered":"2024-09-14T04:02:50.338671+0000","last_clean":"2024-09-14T04:02:50.338671+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:02:50.338671+0000","last_undegraded":"2024-09-14T04:02:50.338671+0000","last_fullsized":"2024-09-14T04:02:50.338671+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918187+0000","last_change":"2024-09-14T04:02:33.187805+0000","last_active":"2024-09-14T04:02:51.918187+0000","last_peered":"2024-09-14T04:02:51.918187+0000","last_clean":"2024-09-14T04:02:51.918187+0000","last_became_active":"2024-09-14T04:02:33.187425+0000","last_became_peered":"2024-09-14T04:02:33.187425+0000","last_unstale":"2024-09-14T04:02:51.918187+0000","last_undegraded":"2024-09-14T04:02:51.918187+0000","last_fullsized":"2024-09-14T04:02:51.918187+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:32.471118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343272+0000","last_change":"2024-09-14T04:02:33.197914+0000","last_active":"2024-09-14T04:02:51.343272+0000","last_peered":"2024-09-14T04:02:51.343272+0000","last_clean":"2024-09-14T04:02:51.343272+0000","last_became_active":"2024-09-14T04:02:33.197447+0000","last_became_peered":"2024-09-14T04:02:33.197447+0000","last_unstale":"2024-09-14T04:02:51.343272+0000","last_undegraded":"2024-09-14T04:02:51.343272+0000","last_fullsized":"2024-09-14T04:02:51.343272+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:08:47.196932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918063+0000","last_change":"2024-09-14T04:02:33.196157+0000","last_active":"2024-09-14T04:02:51.918063+0000","last_peered":"2024-09-14T04:02:51.918063+0000","last_clean":"2024-09-14T04:02:51.918063+0000","last_became_active":"2024-09-14T04:02:33.195682+0000","last_became_peered":"2024-09-14T04:02:33.195682+0000","last_unstale":"2024-09-14T04:02:51.918063+0000","last_undegraded":"2024-09-14T04:02:51.918063+0000","last_fullsized":"2024-09-14T04:02:51.918063+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:04.357130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625500+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:02:51.625500+0000","last_peered":"2024-09-14T04:02:51.625500+0000","last_clean":"2024-09-14T04:02:51.625500+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:02:51.625500+0000","last_undegraded":"2024-09-14T04:02:51.625500+0000","last_fullsized":"2024-09-14T04:02:51.625500+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918026+0000","last_change":"2024-09-14T04:02:33.195513+0000","last_active":"2024-09-14T04:02:51.918026+0000","last_peered":"2024-09-14T04:02:51.918026+0000","last_clean":"2024-09-14T04:02:51.918026+0000","last_became_active":"2024-09-14T04:02:33.195109+0000","last_became_peered":"2024-09-14T04:02:33.195109+0000","last_unstale":"2024-09-14T04:02:51.918026+0000","last_undegraded":"2024-09-14T04:02:51.918026+0000","last_fullsized":"2024-09-14T04:02:51.918026+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:30.591898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342747+0000","last_change":"2024-09-14T04:02:33.198166+0000","last_active":"2024-09-14T04:02:51.342747+0000","last_peered":"2024-09-14T04:02:51.342747+0000","last_clean":"2024-09-14T04:02:51.342747+0000","last_became_active":"2024-09-14T04:02:33.197287+0000","last_became_peered":"2024-09-14T04:02:33.197287+0000","last_unstale":"2024-09-14T04:02:51.342747+0000","last_undegraded":"2024-09-14T04:02:51.342747+0000","last_fullsized":"2024-09-14T04:02:51.342747+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:41:48.137435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342622+0000","last_change":"2024-09-14T04:02:33.192061+0000","last_active":"2024-09-14T04:02:51.342622+0000","last_peered":"2024-09-14T04:02:51.342622+0000","last_clean":"2024-09-14T04:02:51.342622+0000","last_became_active":"2024-09-14T04:02:33.191529+0000","last_became_peered":"2024-09-14T04:02:33.191529+0000","last_unstale":"2024-09-14T04:02:51.342622+0000","last_undegraded":"2024-09-14T04:02:51.342622+0000","last_fullsized":"2024-09-14T04:02:51.342622+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:06.031084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918787+0000","last_change":"2024-09-14T04:02:33.191433+0000","last_active":"2024-09-14T04:02:51.918787+0000","last_peered":"2024-09-14T04:02:51.918787+0000","last_clean":"2024-09-14T04:02:51.918787+0000","last_became_active":"2024-09-14T04:02:33.190554+0000","last_became_peered":"2024-09-14T04:02:33.190554+0000","last_unstale":"2024-09-14T04:02:51.918787+0000","last_undegraded":"2024-09-14T04:02:51.918787+0000","last_fullsized":"2024-09-14T04:02:51.918787+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:58.975784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342433+0000","last_change":"2024-09-14T04:01:59.053348+0000","last_active":"2024-09-14T04:02:51.342433+0000","last_peered":"2024-09-14T04:02:51.342433+0000","last_clean":"2024-09-14T04:02:51.342433+0000","last_became_active":"2024-09-14T04:01:59.053004+0000","last_became_peered":"2024-09-14T04:01:59.053004+0000","last_unstale":"2024-09-14T04:02:51.342433+0000","last_undegraded":"2024-09-14T04:02:51.342433+0000","last_fullsized":"2024-09-14T04:02:51.342433+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:56:28.268868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918249+0000","last_change":"2024-09-14T04:02:33.191341+0000","last_active":"2024-09-14T04:02:51.918249+0000","last_peered":"2024-09-14T04:02:51.918249+0000","last_clean":"2024-09-14T04:02:51.918249+0000","last_became_active":"2024-09-14T04:02:33.190081+0000","last_became_peered":"2024-09-14T04:02:33.190081+0000","last_unstale":"2024-09-14T04:02:51.918249+0000","last_undegraded":"2024-09-14T04:02:51.918249+0000","last_fullsized":"2024-09-14T04:02:51.918249+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:52.740363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342908+0000","last_change":"2024-09-14T04:01:59.057068+0000","last_active":"2024-09-14T04:02:51.342908+0000","last_peered":"2024-09-14T04:02:51.342908+0000","last_clean":"2024-09-14T04:02:51.342908+0000","last_became_active":"2024-09-14T04:01:59.056702+0000","last_became_peered":"2024-09-14T04:01:59.056702+0000","last_unstale":"2024-09-14T04:02:51.342908+0000","last_undegraded":"2024-09-14T04:02:51.342908+0000","last_fullsized":"2024-09-14T04:02:51.342908+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:23.708120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341445+0000","last_change":"2024-09-14T04:02:33.192285+0000","last_active":"2024-09-14T04:02:50.341445+0000","last_peered":"2024-09-14T04:02:50.341445+0000","last_clean":"2024-09-14T04:02:50.341445+0000","last_became_active":"2024-09-14T04:02:33.191832+0000","last_became_peered":"2024-09-14T04:02:33.191832+0000","last_unstale":"2024-09-14T04:02:50.341445+0000","last_undegraded":"2024-09-14T04:02:50.341445+0000","last_fullsized":"2024-09-14T04:02:50.341445+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:38:16.739279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":136,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.346840+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:02:51.346840+0000","last_peered":"2024-09-14T04:02:51.346840+0000","last_clean":"2024-09-14T04:02:51.346840+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:02:51.346840+0000","last_undegraded":"2024-09-14T04:02:51.346840+0000","last_fullsized":"2024-09-14T04:02:51.346840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341322+0000","last_change":"2024-09-14T04:02:33.194384+0000","last_active":"2024-09-14T04:02:50.341322+0000","last_peered":"2024-09-14T04:02:50.341322+0000","last_clean":"2024-09-14T04:02:50.341322+0000","last_became_active":"2024-09-14T04:02:33.194027+0000","last_became_peered":"2024-09-14T04:02:33.194027+0000","last_unstale":"2024-09-14T04:02:50.341322+0000","last_undegraded":"2024-09-14T04:02:50.341322+0000","last_fullsized":"2024-09-14T04:02:50.341322+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:46:29.339591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338808+0000","last_change":"2024-09-14T03:57:08.990606+0000","last_active":"2024-09-14T04:02:50.338808+0000","last_peered":"2024-09-14T04:02:50.338808+0000","last_clean":"2024-09-14T04:02:50.338808+0000","last_became_active":"2024-09-14T03:57:08.979113+0000","last_became_peered":"2024-09-14T03:57:08.979113+0000","last_unstale":"2024-09-14T04:02:50.338808+0000","last_undegraded":"2024-09-14T04:02:50.338808+0000","last_fullsized":"2024-09-14T04:02:50.338808+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:53.542287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918089+0000","last_change":"2024-09-14T04:02:33.194588+0000","last_active":"2024-09-14T04:02:51.918089+0000","last_peered":"2024-09-14T04:02:51.918089+0000","last_clean":"2024-09-14T04:02:51.918089+0000","last_became_active":"2024-09-14T04:02:33.193732+0000","last_became_peered":"2024-09-14T04:02:33.193732+0000","last_unstale":"2024-09-14T04:02:51.918089+0000","last_undegraded":"2024-09-14T04:02:51.918089+0000","last_fullsized":"2024-09-14T04:02:51.918089+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:50.529572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918743+0000","last_change":"2024-09-14T04:02:33.193693+0000","last_active":"2024-09-14T04:02:51.918743+0000","last_peered":"2024-09-14T04:02:51.918743+0000","last_clean":"2024-09-14T04:02:51.918743+0000","last_became_active":"2024-09-14T04:02:33.192884+0000","last_became_peered":"2024-09-14T04:02:33.192884+0000","last_unstale":"2024-09-14T04:02:51.918743+0000","last_undegraded":"2024-09-14T04:02:51.918743+0000","last_fullsized":"2024-09-14T04:02:51.918743+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:21.242232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341118+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:02:50.341118+0000","last_peered":"2024-09-14T04:02:50.341118+0000","last_clean":"2024-09-14T04:02:50.341118+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:02:50.341118+0000","last_undegraded":"2024-09-14T04:02:50.341118+0000","last_fullsized":"2024-09-14T04:02:50.341118+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":102,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343134+0000","last_change":"2024-09-14T04:01:59.054141+0000","last_active":"2024-09-14T04:02:51.343134+0000","last_peered":"2024-09-14T04:02:51.343134+0000","last_clean":"2024-09-14T04:02:51.343134+0000","last_became_active":"2024-09-14T04:01:59.053397+0000","last_became_peered":"2024-09-14T04:01:59.053397+0000","last_unstale":"2024-09-14T04:02:51.343134+0000","last_undegraded":"2024-09-14T04:02:51.343134+0000","last_fullsized":"2024-09-14T04:02:51.343134+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:38.323001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13714,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201554,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466452,"kb_used_data":2184,"kb_used_omap":1,"kb_used_meta":464254,"kb_avail":93278700,"statfs":{"total":95995035648,"available":95517388800,"internally_reserved":0,"allocated":2236416,"data_stored":1440220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":475396555},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463192,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416392,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":93328760,"statfs":{"total":95995035648,"available":95568650240,"internally_reserved":0,"allocated":2224128,"data_stored":1439573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":424147399},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724828,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640580,"kb_used_data":1588,"kb_used_omap":1,"kb_used_meta":638974,"kb_avail":93104572,"statfs":{"total":95995035648,"available":95339081728,"internally_reserved":0,"allocated":1626112,"data_stored":845304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":654309830},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986464,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560208,"kb_used_data":1600,"kb_used_omap":1,"kb_used_meta":558590,"kb_avail":93184944,"statfs":{"total":95995035648,"available":95421382656,"internally_reserved":0,"allocated":1638400,"data_stored":849543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":571996614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748493,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84988,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":83387,"kb_avail":93660164,"statfs":{"total":95995035648,"available":95908007936,"internally_reserved":0,"allocated":1622016,"data_stored":845016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":85388844},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912020,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76036,"kb_used_data":1588,"kb_used_omap":6,"kb_used_meta":74425,"kb_avail":93669116,"statfs":{"total":95995035648,"available":95917174784,"internally_reserved":0,"allocated":1626112,"data_stored":845542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":76211769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468036,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190392,"kb_used_data":1580,"kb_used_omap":5,"kb_used_meta":188794,"kb_avail":93554760,"statfs":{"total":95995035648,"available":95800074240,"internally_reserved":0,"allocated":1617920,"data_stored":843525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":193325407},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":102,"seq":438086664270,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156872,"kb_used_data":2172,"kb_used_omap":5,"kb_used_meta":154682,"kb_avail":93588280,"statfs":{"total":95995035648,"available":95834398720,"internally_reserved":0,"allocated":2224128,"data_stored":1435932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5976,"internal_metadata":158394536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4382,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:03:29.440 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T04:03:29.440 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T04:03:29.440 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T04:03:29.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T04:03:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:29 smithi001 ceph-mon[107399]: from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:29 smithi001 ceph-mon[107399]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:29 smithi001 ceph-mon[112165]: from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:29 smithi001 ceph-mon[112165]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:29 smithi012 ceph-mon[101180]: from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:29 smithi012 ceph-mon[101180]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:30.216 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:31.375 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:31 smithi001 ceph-mon[112165]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:31.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:31 smithi001 ceph-mon[107399]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:31 smithi012 ceph-mon[101180]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:32.155 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:03:32.155 INFO:teuthology.orchestra.run.smithi001.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T04:03:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2241962904' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:03:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2241962904' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:03:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2241962904' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:03:32.859 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T04:03:32.859 INFO:teuthology.run_tasks:Running task print... 2024-09-14T04:03:32.870 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-14T04:03:32.870 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T04:03:32.916 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T04:03:32.916 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T04:03:32.916 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-14T04:03:32.919 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-14T04:03:32.920 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded'}} 2024-09-14T04:03:32.920 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-14T04:03:32.920 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-14T04:03:32.920 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T04:03:32.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:03:33.379 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:34.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[112165]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:34.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.144 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:33 smithi001 ceph-mon[107399]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:33 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:33 smithi012 ceph-mon[101180]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:35.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:35.107 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:36.001 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:03:36.002 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:03:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:36 smithi001 ceph-mon[107399]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:36 smithi001 ceph-mon[112165]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:36 smithi012 ceph-mon[101180]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:36.827 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:36] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T04:03:36.849 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":433,"stamp":"2024-09-14T04:03:35.286074+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771138,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2591920,"kb_used_data":14468,"kb_used_omap":28,"kb_used_meta":2577315,"kb_avail":747369296,"statfs":{"total":767960285184,"available":765306159104,"internally_reserved":0,"allocated":14815232,"data_stored":8544655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29302,"internal_metadata":2639170954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003062"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918602+0000","last_change":"2024-09-14T04:02:33.186418+0000","last_active":"2024-09-14T04:02:51.918602+0000","last_peered":"2024-09-14T04:02:51.918602+0000","last_clean":"2024-09-14T04:02:51.918602+0000","last_became_active":"2024-09-14T04:02:33.186052+0000","last_became_peered":"2024-09-14T04:02:33.186052+0000","last_unstale":"2024-09-14T04:02:51.918602+0000","last_undegraded":"2024-09-14T04:02:51.918602+0000","last_fullsized":"2024-09-14T04:02:51.918602+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:04.240828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":89,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342633+0000","last_change":"2024-09-14T04:01:59.055661+0000","last_active":"2024-09-14T04:02:51.342633+0000","last_peered":"2024-09-14T04:02:51.342633+0000","last_clean":"2024-09-14T04:02:51.342633+0000","last_became_active":"2024-09-14T04:01:59.055301+0000","last_became_peered":"2024-09-14T04:01:59.055301+0000","last_unstale":"2024-09-14T04:02:51.342633+0000","last_undegraded":"2024-09-14T04:02:51.342633+0000","last_fullsized":"2024-09-14T04:02:51.342633+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:34:27.575772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338958+0000","last_change":"2024-09-14T04:02:33.189718+0000","last_active":"2024-09-14T04:02:50.338958+0000","last_peered":"2024-09-14T04:02:50.338958+0000","last_clean":"2024-09-14T04:02:50.338958+0000","last_became_active":"2024-09-14T04:02:33.189342+0000","last_became_peered":"2024-09-14T04:02:33.189342+0000","last_unstale":"2024-09-14T04:02:50.338958+0000","last_undegraded":"2024-09-14T04:02:50.338958+0000","last_fullsized":"2024-09-14T04:02:50.338958+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:51:37.536543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342588+0000","last_change":"2024-09-14T04:02:33.196773+0000","last_active":"2024-09-14T04:02:51.342588+0000","last_peered":"2024-09-14T04:02:51.342588+0000","last_clean":"2024-09-14T04:02:51.342588+0000","last_became_active":"2024-09-14T04:02:33.196447+0000","last_became_peered":"2024-09-14T04:02:33.196447+0000","last_unstale":"2024-09-14T04:02:51.342588+0000","last_undegraded":"2024-09-14T04:02:51.342588+0000","last_fullsized":"2024-09-14T04:02:51.342588+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:51.157998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342796+0000","last_change":"2024-09-14T04:02:33.198492+0000","last_active":"2024-09-14T04:02:51.342796+0000","last_peered":"2024-09-14T04:02:51.342796+0000","last_clean":"2024-09-14T04:02:51.342796+0000","last_became_active":"2024-09-14T04:02:33.197804+0000","last_became_peered":"2024-09-14T04:02:33.197804+0000","last_unstale":"2024-09-14T04:02:51.342796+0000","last_undegraded":"2024-09-14T04:02:51.342796+0000","last_fullsized":"2024-09-14T04:02:51.342796+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:37.842086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342325+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:02:50.342325+0000","last_peered":"2024-09-14T04:02:50.342325+0000","last_clean":"2024-09-14T04:02:50.342325+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:02:50.342325+0000","last_undegraded":"2024-09-14T04:02:50.342325+0000","last_fullsized":"2024-09-14T04:02:50.342325+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343001+0000","last_change":"2024-09-14T03:57:55.641587+0000","last_active":"2024-09-14T04:02:51.343001+0000","last_peered":"2024-09-14T04:02:51.343001+0000","last_clean":"2024-09-14T04:02:51.343001+0000","last_became_active":"2024-09-14T03:57:55.640967+0000","last_became_peered":"2024-09-14T03:57:55.640967+0000","last_unstale":"2024-09-14T04:02:51.343001+0000","last_undegraded":"2024-09-14T04:02:51.343001+0000","last_fullsized":"2024-09-14T04:02:51.343001+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:05.024573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":103,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.181388+0000","last_change":"2024-09-14T03:57:07.969661+0000","last_active":"2024-09-14T04:02:33.181388+0000","last_peered":"2024-09-14T04:02:33.181388+0000","last_clean":"2024-09-14T04:02:33.181388+0000","last_became_active":"2024-09-14T03:57:07.969236+0000","last_became_peered":"2024-09-14T03:57:07.969236+0000","last_unstale":"2024-09-14T04:02:33.181388+0000","last_undegraded":"2024-09-14T04:02:33.181388+0000","last_fullsized":"2024-09-14T04:02:33.181388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:28.685717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.919228+0000","last_change":"2024-09-14T03:57:07.978483+0000","last_active":"2024-09-14T04:02:51.919228+0000","last_peered":"2024-09-14T04:02:51.919228+0000","last_clean":"2024-09-14T04:02:51.919228+0000","last_became_active":"2024-09-14T03:57:07.978076+0000","last_became_peered":"2024-09-14T03:57:07.978076+0000","last_unstale":"2024-09-14T04:02:51.919228+0000","last_undegraded":"2024-09-14T04:02:51.919228+0000","last_fullsized":"2024-09-14T04:02:51.919228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:33.126366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338345+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T04:02:50.338345+0000","last_peered":"2024-09-14T04:02:50.338345+0000","last_clean":"2024-09-14T04:02:50.338345+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T04:02:50.338345+0000","last_undegraded":"2024-09-14T04:02:50.338345+0000","last_fullsized":"2024-09-14T04:02:50.338345+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918439+0000","last_change":"2024-09-14T03:57:55.658263+0000","last_active":"2024-09-14T04:02:51.918439+0000","last_peered":"2024-09-14T04:02:51.918439+0000","last_clean":"2024-09-14T04:02:51.918439+0000","last_became_active":"2024-09-14T03:57:55.657817+0000","last_became_peered":"2024-09-14T03:57:55.657817+0000","last_unstale":"2024-09-14T04:02:51.918439+0000","last_undegraded":"2024-09-14T04:02:51.918439+0000","last_fullsized":"2024-09-14T04:02:51.918439+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:36.196909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341202+0000","last_change":"2024-09-14T04:02:33.196376+0000","last_active":"2024-09-14T04:02:50.341202+0000","last_peered":"2024-09-14T04:02:50.341202+0000","last_clean":"2024-09-14T04:02:50.341202+0000","last_became_active":"2024-09-14T04:02:33.195804+0000","last_became_peered":"2024-09-14T04:02:33.195804+0000","last_unstale":"2024-09-14T04:02:50.341202+0000","last_undegraded":"2024-09-14T04:02:50.341202+0000","last_fullsized":"2024-09-14T04:02:50.341202+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:58:02.150113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918423+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T04:02:51.918423+0000","last_peered":"2024-09-14T04:02:51.918423+0000","last_clean":"2024-09-14T04:02:51.918423+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T04:02:51.918423+0000","last_undegraded":"2024-09-14T04:02:51.918423+0000","last_fullsized":"2024-09-14T04:02:51.918423+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182266+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:02:33.182266+0000","last_peered":"2024-09-14T04:02:33.182266+0000","last_clean":"2024-09-14T04:02:33.182266+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:02:33.182266+0000","last_undegraded":"2024-09-14T04:02:33.182266+0000","last_fullsized":"2024-09-14T04:02:33.182266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.598290+0000","last_change":"2024-09-14T04:02:33.189804+0000","last_active":"2024-09-14T04:02:50.598290+0000","last_peered":"2024-09-14T04:02:50.598290+0000","last_clean":"2024-09-14T04:02:50.598290+0000","last_became_active":"2024-09-14T04:02:33.189427+0000","last_became_peered":"2024-09-14T04:02:33.189427+0000","last_unstale":"2024-09-14T04:02:50.598290+0000","last_undegraded":"2024-09-14T04:02:50.598290+0000","last_fullsized":"2024-09-14T04:02:50.598290+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:35.027998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342724+0000","last_change":"2024-09-14T03:57:55.640069+0000","last_active":"2024-09-14T04:02:51.342724+0000","last_peered":"2024-09-14T04:02:51.342724+0000","last_clean":"2024-09-14T04:02:51.342724+0000","last_became_active":"2024-09-14T03:57:55.639653+0000","last_became_peered":"2024-09-14T03:57:55.639653+0000","last_unstale":"2024-09-14T04:02:51.342724+0000","last_undegraded":"2024-09-14T04:02:51.342724+0000","last_fullsized":"2024-09-14T04:02:51.342724+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:16.579752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918366+0000","last_change":"2024-09-14T04:02:33.189648+0000","last_active":"2024-09-14T04:02:51.918366+0000","last_peered":"2024-09-14T04:02:51.918366+0000","last_clean":"2024-09-14T04:02:51.918366+0000","last_became_active":"2024-09-14T04:02:33.188956+0000","last_became_peered":"2024-09-14T04:02:33.188956+0000","last_unstale":"2024-09-14T04:02:51.918366+0000","last_undegraded":"2024-09-14T04:02:51.918366+0000","last_fullsized":"2024-09-14T04:02:51.918366+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:14.851661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.384607+0000","last_change":"2024-09-14T04:02:33.194407+0000","last_active":"2024-09-14T04:02:50.384607+0000","last_peered":"2024-09-14T04:02:50.384607+0000","last_clean":"2024-09-14T04:02:50.384607+0000","last_became_active":"2024-09-14T04:02:33.194049+0000","last_became_peered":"2024-09-14T04:02:33.194049+0000","last_unstale":"2024-09-14T04:02:50.384607+0000","last_undegraded":"2024-09-14T04:02:50.384607+0000","last_fullsized":"2024-09-14T04:02:50.384607+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:09.389524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182415+0000","last_change":"2024-09-14T03:57:55.680448+0000","last_active":"2024-09-14T04:02:33.182415+0000","last_peered":"2024-09-14T04:02:33.182415+0000","last_clean":"2024-09-14T04:02:33.182415+0000","last_became_active":"2024-09-14T03:57:55.680199+0000","last_became_peered":"2024-09-14T03:57:55.680199+0000","last_unstale":"2024-09-14T04:02:33.182415+0000","last_undegraded":"2024-09-14T04:02:33.182415+0000","last_fullsized":"2024-09-14T04:02:33.182415+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:03.237323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340655+0000","last_change":"2024-09-14T04:01:59.052174+0000","last_active":"2024-09-14T04:02:51.340655+0000","last_peered":"2024-09-14T04:02:51.340655+0000","last_clean":"2024-09-14T04:02:51.340655+0000","last_became_active":"2024-09-14T04:01:59.051831+0000","last_became_peered":"2024-09-14T04:01:59.051831+0000","last_unstale":"2024-09-14T04:02:51.340655+0000","last_undegraded":"2024-09-14T04:02:51.340655+0000","last_fullsized":"2024-09-14T04:02:51.340655+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:15.224478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341485+0000","last_change":"2024-09-14T04:01:59.055460+0000","last_active":"2024-09-14T04:02:50.341485+0000","last_peered":"2024-09-14T04:02:50.341485+0000","last_clean":"2024-09-14T04:02:50.341485+0000","last_became_active":"2024-09-14T04:01:59.055064+0000","last_became_peered":"2024-09-14T04:01:59.055064+0000","last_unstale":"2024-09-14T04:02:50.341485+0000","last_undegraded":"2024-09-14T04:02:50.341485+0000","last_fullsized":"2024-09-14T04:02:50.341485+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:52.089107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340265+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:02:51.340265+0000","last_peered":"2024-09-14T04:02:51.340265+0000","last_clean":"2024-09-14T04:02:51.340265+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:02:51.340265+0000","last_undegraded":"2024-09-14T04:02:51.340265+0000","last_fullsized":"2024-09-14T04:02:51.340265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917955+0000","last_change":"2024-09-14T04:02:33.194317+0000","last_active":"2024-09-14T04:02:51.917955+0000","last_peered":"2024-09-14T04:02:51.917955+0000","last_clean":"2024-09-14T04:02:51.917955+0000","last_became_active":"2024-09-14T04:02:33.193981+0000","last_became_peered":"2024-09-14T04:02:33.193981+0000","last_unstale":"2024-09-14T04:02:51.917955+0000","last_undegraded":"2024-09-14T04:02:51.917955+0000","last_fullsized":"2024-09-14T04:02:51.917955+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:35:21.725699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343573+0000","last_change":"2024-09-14T04:02:33.192562+0000","last_active":"2024-09-14T04:02:51.343573+0000","last_peered":"2024-09-14T04:02:51.343573+0000","last_clean":"2024-09-14T04:02:51.343573+0000","last_became_active":"2024-09-14T04:02:33.192185+0000","last_became_peered":"2024-09-14T04:02:33.192185+0000","last_unstale":"2024-09-14T04:02:51.343573+0000","last_undegraded":"2024-09-14T04:02:51.343573+0000","last_fullsized":"2024-09-14T04:02:51.343573+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:34:52.201865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":429,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918623+0000","last_change":"2024-09-14T03:57:07.974148+0000","last_active":"2024-09-14T04:02:51.918623+0000","last_peered":"2024-09-14T04:02:51.918623+0000","last_clean":"2024-09-14T04:02:51.918623+0000","last_became_active":"2024-09-14T03:57:07.973612+0000","last_became_peered":"2024-09-14T03:57:07.973612+0000","last_unstale":"2024-09-14T04:02:51.918623+0000","last_undegraded":"2024-09-14T04:02:51.918623+0000","last_fullsized":"2024-09-14T04:02:51.918623+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:13.349866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338527+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:02:50.338527+0000","last_peered":"2024-09-14T04:02:50.338527+0000","last_clean":"2024-09-14T04:02:50.338527+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:02:50.338527+0000","last_undegraded":"2024-09-14T04:02:50.338527+0000","last_fullsized":"2024-09-14T04:02:50.338527+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918082+0000","last_change":"2024-09-14T04:02:33.189844+0000","last_active":"2024-09-14T04:02:51.918082+0000","last_peered":"2024-09-14T04:02:51.918082+0000","last_clean":"2024-09-14T04:02:51.918082+0000","last_became_active":"2024-09-14T04:02:33.189492+0000","last_became_peered":"2024-09-14T04:02:33.189492+0000","last_unstale":"2024-09-14T04:02:51.918082+0000","last_undegraded":"2024-09-14T04:02:51.918082+0000","last_fullsized":"2024-09-14T04:02:51.918082+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:28.814763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182854+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:02:33.182854+0000","last_peered":"2024-09-14T04:02:33.182854+0000","last_clean":"2024-09-14T04:02:33.182854+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:02:33.182854+0000","last_undegraded":"2024-09-14T04:02:33.182854+0000","last_fullsized":"2024-09-14T04:02:33.182854+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918284+0000","last_change":"2024-09-14T03:57:55.681501+0000","last_active":"2024-09-14T04:02:51.918284+0000","last_peered":"2024-09-14T04:02:51.918284+0000","last_clean":"2024-09-14T04:02:51.918284+0000","last_became_active":"2024-09-14T03:57:55.681104+0000","last_became_peered":"2024-09-14T03:57:55.681104+0000","last_unstale":"2024-09-14T04:02:51.918284+0000","last_undegraded":"2024-09-14T04:02:51.918284+0000","last_fullsized":"2024-09-14T04:02:51.918284+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:54.441046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342257+0000","last_change":"2024-09-14T04:02:33.189629+0000","last_active":"2024-09-14T04:02:51.342257+0000","last_peered":"2024-09-14T04:02:51.342257+0000","last_clean":"2024-09-14T04:02:51.342257+0000","last_became_active":"2024-09-14T04:02:33.189317+0000","last_became_peered":"2024-09-14T04:02:33.189317+0000","last_unstale":"2024-09-14T04:02:51.342257+0000","last_undegraded":"2024-09-14T04:02:51.342257+0000","last_fullsized":"2024-09-14T04:02:51.342257+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:09.638142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342624+0000","last_change":"2024-09-14T04:01:59.060194+0000","last_active":"2024-09-14T04:02:51.342624+0000","last_peered":"2024-09-14T04:02:51.342624+0000","last_clean":"2024-09-14T04:02:51.342624+0000","last_became_active":"2024-09-14T04:01:59.059877+0000","last_became_peered":"2024-09-14T04:01:59.059877+0000","last_unstale":"2024-09-14T04:02:51.342624+0000","last_undegraded":"2024-09-14T04:02:51.342624+0000","last_fullsized":"2024-09-14T04:02:51.342624+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:34.061157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918185+0000","last_change":"2024-09-14T04:02:33.186950+0000","last_active":"2024-09-14T04:02:51.918185+0000","last_peered":"2024-09-14T04:02:51.918185+0000","last_clean":"2024-09-14T04:02:51.918185+0000","last_became_active":"2024-09-14T04:02:33.186615+0000","last_became_peered":"2024-09-14T04:02:33.186615+0000","last_unstale":"2024-09-14T04:02:51.918185+0000","last_undegraded":"2024-09-14T04:02:51.918185+0000","last_fullsized":"2024-09-14T04:02:51.918185+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:26.114627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342698+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:02:51.342698+0000","last_peered":"2024-09-14T04:02:51.342698+0000","last_clean":"2024-09-14T04:02:51.342698+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:02:51.342698+0000","last_undegraded":"2024-09-14T04:02:51.342698+0000","last_fullsized":"2024-09-14T04:02:51.342698+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343360+0000","last_change":"2024-09-14T03:57:55.641841+0000","last_active":"2024-09-14T04:02:51.343360+0000","last_peered":"2024-09-14T04:02:51.343360+0000","last_clean":"2024-09-14T04:02:51.343360+0000","last_became_active":"2024-09-14T03:57:55.638515+0000","last_became_peered":"2024-09-14T03:57:55.638515+0000","last_unstale":"2024-09-14T04:02:51.343360+0000","last_undegraded":"2024-09-14T04:02:51.343360+0000","last_fullsized":"2024-09-14T04:02:51.343360+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:07.687766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":113,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338437+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:02:50.338437+0000","last_peered":"2024-09-14T04:02:50.338437+0000","last_clean":"2024-09-14T04:02:50.338437+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:02:50.338437+0000","last_undegraded":"2024-09-14T04:02:50.338437+0000","last_fullsized":"2024-09-14T04:02:50.338437+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340604+0000","last_change":"2024-09-14T03:57:07.985126+0000","last_active":"2024-09-14T04:02:51.340604+0000","last_peered":"2024-09-14T04:02:51.340604+0000","last_clean":"2024-09-14T04:02:51.340604+0000","last_became_active":"2024-09-14T03:57:07.984427+0000","last_became_peered":"2024-09-14T03:57:07.984427+0000","last_unstale":"2024-09-14T04:02:51.340604+0000","last_undegraded":"2024-09-14T04:02:51.340604+0000","last_fullsized":"2024-09-14T04:02:51.340604+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:23.230268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338533+0000","last_change":"2024-09-14T04:02:33.189850+0000","last_active":"2024-09-14T04:02:50.338533+0000","last_peered":"2024-09-14T04:02:50.338533+0000","last_clean":"2024-09-14T04:02:50.338533+0000","last_became_active":"2024-09-14T04:02:33.189444+0000","last_became_peered":"2024-09-14T04:02:33.189444+0000","last_unstale":"2024-09-14T04:02:50.338533+0000","last_undegraded":"2024-09-14T04:02:50.338533+0000","last_fullsized":"2024-09-14T04:02:50.338533+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:23.120760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338905+0000","last_change":"2024-09-14T04:01:59.052874+0000","last_active":"2024-09-14T04:02:50.338905+0000","last_peered":"2024-09-14T04:02:50.338905+0000","last_clean":"2024-09-14T04:02:50.338905+0000","last_became_active":"2024-09-14T04:01:59.052455+0000","last_became_peered":"2024-09-14T04:01:59.052455+0000","last_unstale":"2024-09-14T04:02:50.338905+0000","last_undegraded":"2024-09-14T04:02:50.338905+0000","last_fullsized":"2024-09-14T04:02:50.338905+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:05.518917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343376+0000","last_change":"2024-09-14T03:57:55.656097+0000","last_active":"2024-09-14T04:02:51.343376+0000","last_peered":"2024-09-14T04:02:51.343376+0000","last_clean":"2024-09-14T04:02:51.343376+0000","last_became_active":"2024-09-14T03:57:55.649947+0000","last_became_peered":"2024-09-14T03:57:55.649947+0000","last_unstale":"2024-09-14T04:02:51.343376+0000","last_undegraded":"2024-09-14T04:02:51.343376+0000","last_fullsized":"2024-09-14T04:02:51.343376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.222621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342967+0000","last_change":"2024-09-14T03:57:55.667306+0000","last_active":"2024-09-14T04:02:51.342967+0000","last_peered":"2024-09-14T04:02:51.342967+0000","last_clean":"2024-09-14T04:02:51.342967+0000","last_became_active":"2024-09-14T03:57:55.661348+0000","last_became_peered":"2024-09-14T03:57:55.661348+0000","last_unstale":"2024-09-14T04:02:51.342967+0000","last_undegraded":"2024-09-14T04:02:51.342967+0000","last_fullsized":"2024-09-14T04:02:51.342967+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:25.408014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343274+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:02:51.343274+0000","last_peered":"2024-09-14T04:02:51.343274+0000","last_clean":"2024-09-14T04:02:51.343274+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:02:51.343274+0000","last_undegraded":"2024-09-14T04:02:51.343274+0000","last_fullsized":"2024-09-14T04:02:51.343274+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342983+0000","last_change":"2024-09-14T03:57:07.974119+0000","last_active":"2024-09-14T04:02:50.342983+0000","last_peered":"2024-09-14T04:02:50.342983+0000","last_clean":"2024-09-14T04:02:50.342983+0000","last_became_active":"2024-09-14T03:57:07.973617+0000","last_became_peered":"2024-09-14T03:57:07.973617+0000","last_unstale":"2024-09-14T04:02:50.342983+0000","last_undegraded":"2024-09-14T04:02:50.342983+0000","last_fullsized":"2024-09-14T04:02:50.342983+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:55.426883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342856+0000","last_change":"2024-09-14T03:57:07.984238+0000","last_active":"2024-09-14T04:02:51.342856+0000","last_peered":"2024-09-14T04:02:51.342856+0000","last_clean":"2024-09-14T04:02:51.342856+0000","last_became_active":"2024-09-14T03:57:07.983677+0000","last_became_peered":"2024-09-14T03:57:07.983677+0000","last_unstale":"2024-09-14T04:02:51.342856+0000","last_undegraded":"2024-09-14T04:02:51.342856+0000","last_fullsized":"2024-09-14T04:02:51.342856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:44:40.642082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918492+0000","last_change":"2024-09-14T04:02:33.190590+0000","last_active":"2024-09-14T04:02:51.918492+0000","last_peered":"2024-09-14T04:02:51.918492+0000","last_clean":"2024-09-14T04:02:51.918492+0000","last_became_active":"2024-09-14T04:02:33.190109+0000","last_became_peered":"2024-09-14T04:02:33.190109+0000","last_unstale":"2024-09-14T04:02:51.918492+0000","last_undegraded":"2024-09-14T04:02:51.918492+0000","last_fullsized":"2024-09-14T04:02:51.918492+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:53:10.464962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343631+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:02:51.343631+0000","last_peered":"2024-09-14T04:02:51.343631+0000","last_clean":"2024-09-14T04:02:51.343631+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:02:51.343631+0000","last_undegraded":"2024-09-14T04:02:51.343631+0000","last_fullsized":"2024-09-14T04:02:51.343631+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917900+0000","last_change":"2024-09-14T04:01:59.057227+0000","last_active":"2024-09-14T04:02:51.917900+0000","last_peered":"2024-09-14T04:02:51.917900+0000","last_clean":"2024-09-14T04:02:51.917900+0000","last_became_active":"2024-09-14T04:01:59.056799+0000","last_became_peered":"2024-09-14T04:01:59.056799+0000","last_unstale":"2024-09-14T04:02:51.917900+0000","last_undegraded":"2024-09-14T04:02:51.917900+0000","last_fullsized":"2024-09-14T04:02:51.917900+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:54.217497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":101,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342893+0000","last_change":"2024-09-14T03:57:55.649322+0000","last_active":"2024-09-14T04:02:51.342893+0000","last_peered":"2024-09-14T04:02:51.342893+0000","last_clean":"2024-09-14T04:02:51.342893+0000","last_became_active":"2024-09-14T03:57:55.644934+0000","last_became_peered":"2024-09-14T03:57:55.644934+0000","last_unstale":"2024-09-14T04:02:51.342893+0000","last_undegraded":"2024-09-14T04:02:51.342893+0000","last_fullsized":"2024-09-14T04:02:51.342893+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:35.277557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625457+0000","last_change":"2024-09-14T04:01:59.055291+0000","last_active":"2024-09-14T04:02:51.625457+0000","last_peered":"2024-09-14T04:02:51.625457+0000","last_clean":"2024-09-14T04:02:51.625457+0000","last_became_active":"2024-09-14T04:01:59.054919+0000","last_became_peered":"2024-09-14T04:01:59.054919+0000","last_unstale":"2024-09-14T04:02:51.625457+0000","last_undegraded":"2024-09-14T04:02:51.625457+0000","last_fullsized":"2024-09-14T04:02:51.625457+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:22.638368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340137+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:02:51.340137+0000","last_peered":"2024-09-14T04:02:51.340137+0000","last_clean":"2024-09-14T04:02:51.340137+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:02:51.340137+0000","last_undegraded":"2024-09-14T04:02:51.340137+0000","last_fullsized":"2024-09-14T04:02:51.340137+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341483+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:02:50.341483+0000","last_peered":"2024-09-14T04:02:50.341483+0000","last_clean":"2024-09-14T04:02:50.341483+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:02:50.341483+0000","last_undegraded":"2024-09-14T04:02:50.341483+0000","last_fullsized":"2024-09-14T04:02:50.341483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341694+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:02:50.341694+0000","last_peered":"2024-09-14T04:02:50.341694+0000","last_clean":"2024-09-14T04:02:50.341694+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:02:50.341694+0000","last_undegraded":"2024-09-14T04:02:50.341694+0000","last_fullsized":"2024-09-14T04:02:50.341694+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342812+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:02:51.342812+0000","last_peered":"2024-09-14T04:02:51.342812+0000","last_clean":"2024-09-14T04:02:51.342812+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:02:51.342812+0000","last_undegraded":"2024-09-14T04:02:51.342812+0000","last_fullsized":"2024-09-14T04:02:51.342812+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342777+0000","last_change":"2024-09-14T04:02:33.192294+0000","last_active":"2024-09-14T04:02:50.342777+0000","last_peered":"2024-09-14T04:02:50.342777+0000","last_clean":"2024-09-14T04:02:50.342777+0000","last_became_active":"2024-09-14T04:02:33.191886+0000","last_became_peered":"2024-09-14T04:02:33.191886+0000","last_unstale":"2024-09-14T04:02:50.342777+0000","last_undegraded":"2024-09-14T04:02:50.342777+0000","last_fullsized":"2024-09-14T04:02:50.342777+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:37:31.901824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338671+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:02:50.338671+0000","last_peered":"2024-09-14T04:02:50.338671+0000","last_clean":"2024-09-14T04:02:50.338671+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:02:50.338671+0000","last_undegraded":"2024-09-14T04:02:50.338671+0000","last_fullsized":"2024-09-14T04:02:50.338671+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918187+0000","last_change":"2024-09-14T04:02:33.187805+0000","last_active":"2024-09-14T04:02:51.918187+0000","last_peered":"2024-09-14T04:02:51.918187+0000","last_clean":"2024-09-14T04:02:51.918187+0000","last_became_active":"2024-09-14T04:02:33.187425+0000","last_became_peered":"2024-09-14T04:02:33.187425+0000","last_unstale":"2024-09-14T04:02:51.918187+0000","last_undegraded":"2024-09-14T04:02:51.918187+0000","last_fullsized":"2024-09-14T04:02:51.918187+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:32.471118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343272+0000","last_change":"2024-09-14T04:02:33.197914+0000","last_active":"2024-09-14T04:02:51.343272+0000","last_peered":"2024-09-14T04:02:51.343272+0000","last_clean":"2024-09-14T04:02:51.343272+0000","last_became_active":"2024-09-14T04:02:33.197447+0000","last_became_peered":"2024-09-14T04:02:33.197447+0000","last_unstale":"2024-09-14T04:02:51.343272+0000","last_undegraded":"2024-09-14T04:02:51.343272+0000","last_fullsized":"2024-09-14T04:02:51.343272+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:08:47.196932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918063+0000","last_change":"2024-09-14T04:02:33.196157+0000","last_active":"2024-09-14T04:02:51.918063+0000","last_peered":"2024-09-14T04:02:51.918063+0000","last_clean":"2024-09-14T04:02:51.918063+0000","last_became_active":"2024-09-14T04:02:33.195682+0000","last_became_peered":"2024-09-14T04:02:33.195682+0000","last_unstale":"2024-09-14T04:02:51.918063+0000","last_undegraded":"2024-09-14T04:02:51.918063+0000","last_fullsized":"2024-09-14T04:02:51.918063+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:04.357130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625500+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:02:51.625500+0000","last_peered":"2024-09-14T04:02:51.625500+0000","last_clean":"2024-09-14T04:02:51.625500+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:02:51.625500+0000","last_undegraded":"2024-09-14T04:02:51.625500+0000","last_fullsized":"2024-09-14T04:02:51.625500+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918026+0000","last_change":"2024-09-14T04:02:33.195513+0000","last_active":"2024-09-14T04:02:51.918026+0000","last_peered":"2024-09-14T04:02:51.918026+0000","last_clean":"2024-09-14T04:02:51.918026+0000","last_became_active":"2024-09-14T04:02:33.195109+0000","last_became_peered":"2024-09-14T04:02:33.195109+0000","last_unstale":"2024-09-14T04:02:51.918026+0000","last_undegraded":"2024-09-14T04:02:51.918026+0000","last_fullsized":"2024-09-14T04:02:51.918026+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:30.591898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342747+0000","last_change":"2024-09-14T04:02:33.198166+0000","last_active":"2024-09-14T04:02:51.342747+0000","last_peered":"2024-09-14T04:02:51.342747+0000","last_clean":"2024-09-14T04:02:51.342747+0000","last_became_active":"2024-09-14T04:02:33.197287+0000","last_became_peered":"2024-09-14T04:02:33.197287+0000","last_unstale":"2024-09-14T04:02:51.342747+0000","last_undegraded":"2024-09-14T04:02:51.342747+0000","last_fullsized":"2024-09-14T04:02:51.342747+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:41:48.137435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342622+0000","last_change":"2024-09-14T04:02:33.192061+0000","last_active":"2024-09-14T04:02:51.342622+0000","last_peered":"2024-09-14T04:02:51.342622+0000","last_clean":"2024-09-14T04:02:51.342622+0000","last_became_active":"2024-09-14T04:02:33.191529+0000","last_became_peered":"2024-09-14T04:02:33.191529+0000","last_unstale":"2024-09-14T04:02:51.342622+0000","last_undegraded":"2024-09-14T04:02:51.342622+0000","last_fullsized":"2024-09-14T04:02:51.342622+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:06.031084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918787+0000","last_change":"2024-09-14T04:02:33.191433+0000","last_active":"2024-09-14T04:02:51.918787+0000","last_peered":"2024-09-14T04:02:51.918787+0000","last_clean":"2024-09-14T04:02:51.918787+0000","last_became_active":"2024-09-14T04:02:33.190554+0000","last_became_peered":"2024-09-14T04:02:33.190554+0000","last_unstale":"2024-09-14T04:02:51.918787+0000","last_undegraded":"2024-09-14T04:02:51.918787+0000","last_fullsized":"2024-09-14T04:02:51.918787+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:58.975784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342433+0000","last_change":"2024-09-14T04:01:59.053348+0000","last_active":"2024-09-14T04:02:51.342433+0000","last_peered":"2024-09-14T04:02:51.342433+0000","last_clean":"2024-09-14T04:02:51.342433+0000","last_became_active":"2024-09-14T04:01:59.053004+0000","last_became_peered":"2024-09-14T04:01:59.053004+0000","last_unstale":"2024-09-14T04:02:51.342433+0000","last_undegraded":"2024-09-14T04:02:51.342433+0000","last_fullsized":"2024-09-14T04:02:51.342433+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:56:28.268868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918249+0000","last_change":"2024-09-14T04:02:33.191341+0000","last_active":"2024-09-14T04:02:51.918249+0000","last_peered":"2024-09-14T04:02:51.918249+0000","last_clean":"2024-09-14T04:02:51.918249+0000","last_became_active":"2024-09-14T04:02:33.190081+0000","last_became_peered":"2024-09-14T04:02:33.190081+0000","last_unstale":"2024-09-14T04:02:51.918249+0000","last_undegraded":"2024-09-14T04:02:51.918249+0000","last_fullsized":"2024-09-14T04:02:51.918249+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:52.740363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342908+0000","last_change":"2024-09-14T04:01:59.057068+0000","last_active":"2024-09-14T04:02:51.342908+0000","last_peered":"2024-09-14T04:02:51.342908+0000","last_clean":"2024-09-14T04:02:51.342908+0000","last_became_active":"2024-09-14T04:01:59.056702+0000","last_became_peered":"2024-09-14T04:01:59.056702+0000","last_unstale":"2024-09-14T04:02:51.342908+0000","last_undegraded":"2024-09-14T04:02:51.342908+0000","last_fullsized":"2024-09-14T04:02:51.342908+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:23.708120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341445+0000","last_change":"2024-09-14T04:02:33.192285+0000","last_active":"2024-09-14T04:02:50.341445+0000","last_peered":"2024-09-14T04:02:50.341445+0000","last_clean":"2024-09-14T04:02:50.341445+0000","last_became_active":"2024-09-14T04:02:33.191832+0000","last_became_peered":"2024-09-14T04:02:33.191832+0000","last_unstale":"2024-09-14T04:02:50.341445+0000","last_undegraded":"2024-09-14T04:02:50.341445+0000","last_fullsized":"2024-09-14T04:02:50.341445+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:38:16.739279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":136,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.346840+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:02:51.346840+0000","last_peered":"2024-09-14T04:02:51.346840+0000","last_clean":"2024-09-14T04:02:51.346840+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:02:51.346840+0000","last_undegraded":"2024-09-14T04:02:51.346840+0000","last_fullsized":"2024-09-14T04:02:51.346840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341322+0000","last_change":"2024-09-14T04:02:33.194384+0000","last_active":"2024-09-14T04:02:50.341322+0000","last_peered":"2024-09-14T04:02:50.341322+0000","last_clean":"2024-09-14T04:02:50.341322+0000","last_became_active":"2024-09-14T04:02:33.194027+0000","last_became_peered":"2024-09-14T04:02:33.194027+0000","last_unstale":"2024-09-14T04:02:50.341322+0000","last_undegraded":"2024-09-14T04:02:50.341322+0000","last_fullsized":"2024-09-14T04:02:50.341322+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:46:29.339591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338808+0000","last_change":"2024-09-14T03:57:08.990606+0000","last_active":"2024-09-14T04:02:50.338808+0000","last_peered":"2024-09-14T04:02:50.338808+0000","last_clean":"2024-09-14T04:02:50.338808+0000","last_became_active":"2024-09-14T03:57:08.979113+0000","last_became_peered":"2024-09-14T03:57:08.979113+0000","last_unstale":"2024-09-14T04:02:50.338808+0000","last_undegraded":"2024-09-14T04:02:50.338808+0000","last_fullsized":"2024-09-14T04:02:50.338808+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:53.542287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918089+0000","last_change":"2024-09-14T04:02:33.194588+0000","last_active":"2024-09-14T04:02:51.918089+0000","last_peered":"2024-09-14T04:02:51.918089+0000","last_clean":"2024-09-14T04:02:51.918089+0000","last_became_active":"2024-09-14T04:02:33.193732+0000","last_became_peered":"2024-09-14T04:02:33.193732+0000","last_unstale":"2024-09-14T04:02:51.918089+0000","last_undegraded":"2024-09-14T04:02:51.918089+0000","last_fullsized":"2024-09-14T04:02:51.918089+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:50.529572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918743+0000","last_change":"2024-09-14T04:02:33.193693+0000","last_active":"2024-09-14T04:02:51.918743+0000","last_peered":"2024-09-14T04:02:51.918743+0000","last_clean":"2024-09-14T04:02:51.918743+0000","last_became_active":"2024-09-14T04:02:33.192884+0000","last_became_peered":"2024-09-14T04:02:33.192884+0000","last_unstale":"2024-09-14T04:02:51.918743+0000","last_undegraded":"2024-09-14T04:02:51.918743+0000","last_fullsized":"2024-09-14T04:02:51.918743+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:21.242232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341118+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:02:50.341118+0000","last_peered":"2024-09-14T04:02:50.341118+0000","last_clean":"2024-09-14T04:02:50.341118+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:02:50.341118+0000","last_undegraded":"2024-09-14T04:02:50.341118+0000","last_fullsized":"2024-09-14T04:02:50.341118+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":102,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343134+0000","last_change":"2024-09-14T04:01:59.054141+0000","last_active":"2024-09-14T04:02:51.343134+0000","last_peered":"2024-09-14T04:02:51.343134+0000","last_clean":"2024-09-14T04:02:51.343134+0000","last_became_active":"2024-09-14T04:01:59.053397+0000","last_became_peered":"2024-09-14T04:01:59.053397+0000","last_unstale":"2024-09-14T04:02:51.343134+0000","last_undegraded":"2024-09-14T04:02:51.343134+0000","last_fullsized":"2024-09-14T04:02:51.343134+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:38.323001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13714,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201555,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466452,"kb_used_data":2184,"kb_used_omap":1,"kb_used_meta":464254,"kb_avail":93278700,"statfs":{"total":95995035648,"available":95517388800,"internally_reserved":0,"allocated":2236416,"data_stored":1440220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":475396555},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463193,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416392,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":93328760,"statfs":{"total":95995035648,"available":95568650240,"internally_reserved":0,"allocated":2224128,"data_stored":1439573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":424147399},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724829,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640580,"kb_used_data":1588,"kb_used_omap":1,"kb_used_meta":638974,"kb_avail":93104572,"statfs":{"total":95995035648,"available":95339081728,"internally_reserved":0,"allocated":1626112,"data_stored":845304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":654309830},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986465,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560208,"kb_used_data":1600,"kb_used_omap":1,"kb_used_meta":558590,"kb_avail":93184944,"statfs":{"total":95995035648,"available":95421382656,"internally_reserved":0,"allocated":1638400,"data_stored":849543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":571996614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748494,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84988,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":83387,"kb_avail":93660164,"statfs":{"total":95995035648,"available":95908007936,"internally_reserved":0,"allocated":1622016,"data_stored":845016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":85388844},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912021,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76036,"kb_used_data":1588,"kb_used_omap":6,"kb_used_meta":74425,"kb_avail":93669116,"statfs":{"total":95995035648,"available":95917174784,"internally_reserved":0,"allocated":1626112,"data_stored":845542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":76211769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468038,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190392,"kb_used_data":1580,"kb_used_omap":5,"kb_used_meta":188794,"kb_avail":93554760,"statfs":{"total":95995035648,"available":95800074240,"internally_reserved":0,"allocated":1617920,"data_stored":843525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":193325407},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":102,"seq":438086664271,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156872,"kb_used_data":2172,"kb_used_omap":5,"kb_used_meta":154682,"kb_avail":93588280,"statfs":{"total":95995035648,"available":95834398720,"internally_reserved":0,"allocated":2224128,"data_stored":1435932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5976,"internal_metadata":158394536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4382,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:03:36.852 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:03:37.071 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:03:38.577 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:38 smithi001 ceph-mon[107399]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:38.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:38 smithi001 ceph-mon[112165]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:38 smithi012 ceph-mon[101180]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:03:38.986 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:03:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:39 smithi012 ceph-mon[101180]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:39 smithi012 ceph-mon[101180]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:39.643 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:39 smithi001 ceph-mon[107399]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:39.643 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:39 smithi001 ceph-mon[107399]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:39.644 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:39 smithi001 ceph-mon[112165]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:03:39.644 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:39 smithi001 ceph-mon[112165]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:39.665 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":434,"stamp":"2024-09-14T04:03:37.286454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771138,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2591920,"kb_used_data":14468,"kb_used_omap":28,"kb_used_meta":2577315,"kb_avail":747369296,"statfs":{"total":767960285184,"available":765306159104,"internally_reserved":0,"allocated":14815232,"data_stored":8544655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29302,"internal_metadata":2639170954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002894"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918602+0000","last_change":"2024-09-14T04:02:33.186418+0000","last_active":"2024-09-14T04:02:51.918602+0000","last_peered":"2024-09-14T04:02:51.918602+0000","last_clean":"2024-09-14T04:02:51.918602+0000","last_became_active":"2024-09-14T04:02:33.186052+0000","last_became_peered":"2024-09-14T04:02:33.186052+0000","last_unstale":"2024-09-14T04:02:51.918602+0000","last_undegraded":"2024-09-14T04:02:51.918602+0000","last_fullsized":"2024-09-14T04:02:51.918602+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:04.240828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":89,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342633+0000","last_change":"2024-09-14T04:01:59.055661+0000","last_active":"2024-09-14T04:02:51.342633+0000","last_peered":"2024-09-14T04:02:51.342633+0000","last_clean":"2024-09-14T04:02:51.342633+0000","last_became_active":"2024-09-14T04:01:59.055301+0000","last_became_peered":"2024-09-14T04:01:59.055301+0000","last_unstale":"2024-09-14T04:02:51.342633+0000","last_undegraded":"2024-09-14T04:02:51.342633+0000","last_fullsized":"2024-09-14T04:02:51.342633+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:34:27.575772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338958+0000","last_change":"2024-09-14T04:02:33.189718+0000","last_active":"2024-09-14T04:02:50.338958+0000","last_peered":"2024-09-14T04:02:50.338958+0000","last_clean":"2024-09-14T04:02:50.338958+0000","last_became_active":"2024-09-14T04:02:33.189342+0000","last_became_peered":"2024-09-14T04:02:33.189342+0000","last_unstale":"2024-09-14T04:02:50.338958+0000","last_undegraded":"2024-09-14T04:02:50.338958+0000","last_fullsized":"2024-09-14T04:02:50.338958+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:51:37.536543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342588+0000","last_change":"2024-09-14T04:02:33.196773+0000","last_active":"2024-09-14T04:02:51.342588+0000","last_peered":"2024-09-14T04:02:51.342588+0000","last_clean":"2024-09-14T04:02:51.342588+0000","last_became_active":"2024-09-14T04:02:33.196447+0000","last_became_peered":"2024-09-14T04:02:33.196447+0000","last_unstale":"2024-09-14T04:02:51.342588+0000","last_undegraded":"2024-09-14T04:02:51.342588+0000","last_fullsized":"2024-09-14T04:02:51.342588+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:51.157998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342796+0000","last_change":"2024-09-14T04:02:33.198492+0000","last_active":"2024-09-14T04:02:51.342796+0000","last_peered":"2024-09-14T04:02:51.342796+0000","last_clean":"2024-09-14T04:02:51.342796+0000","last_became_active":"2024-09-14T04:02:33.197804+0000","last_became_peered":"2024-09-14T04:02:33.197804+0000","last_unstale":"2024-09-14T04:02:51.342796+0000","last_undegraded":"2024-09-14T04:02:51.342796+0000","last_fullsized":"2024-09-14T04:02:51.342796+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:37.842086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918323+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:02:51.918323+0000","last_peered":"2024-09-14T04:02:51.918323+0000","last_clean":"2024-09-14T04:02:51.918323+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:02:51.918323+0000","last_undegraded":"2024-09-14T04:02:51.918323+0000","last_fullsized":"2024-09-14T04:02:51.918323+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343001+0000","last_change":"2024-09-14T03:57:55.641587+0000","last_active":"2024-09-14T04:02:51.343001+0000","last_peered":"2024-09-14T04:02:51.343001+0000","last_clean":"2024-09-14T04:02:51.343001+0000","last_became_active":"2024-09-14T03:57:55.640967+0000","last_became_peered":"2024-09-14T03:57:55.640967+0000","last_unstale":"2024-09-14T04:02:51.343001+0000","last_undegraded":"2024-09-14T04:02:51.343001+0000","last_fullsized":"2024-09-14T04:02:51.343001+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:05.024573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":104,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625155+0000","last_change":"2024-09-14T03:57:07.969661+0000","last_active":"2024-09-14T04:02:51.625155+0000","last_peered":"2024-09-14T04:02:51.625155+0000","last_clean":"2024-09-14T04:02:51.625155+0000","last_became_active":"2024-09-14T03:57:07.969236+0000","last_became_peered":"2024-09-14T03:57:07.969236+0000","last_unstale":"2024-09-14T04:02:51.625155+0000","last_undegraded":"2024-09-14T04:02:51.625155+0000","last_fullsized":"2024-09-14T04:02:51.625155+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:28.685717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.919228+0000","last_change":"2024-09-14T03:57:07.978483+0000","last_active":"2024-09-14T04:02:51.919228+0000","last_peered":"2024-09-14T04:02:51.919228+0000","last_clean":"2024-09-14T04:02:51.919228+0000","last_became_active":"2024-09-14T03:57:07.978076+0000","last_became_peered":"2024-09-14T03:57:07.978076+0000","last_unstale":"2024-09-14T04:02:51.919228+0000","last_undegraded":"2024-09-14T04:02:51.919228+0000","last_fullsized":"2024-09-14T04:02:51.919228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:33.126366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338345+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T04:02:50.338345+0000","last_peered":"2024-09-14T04:02:50.338345+0000","last_clean":"2024-09-14T04:02:50.338345+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T04:02:50.338345+0000","last_undegraded":"2024-09-14T04:02:50.338345+0000","last_fullsized":"2024-09-14T04:02:50.338345+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918439+0000","last_change":"2024-09-14T03:57:55.658263+0000","last_active":"2024-09-14T04:02:51.918439+0000","last_peered":"2024-09-14T04:02:51.918439+0000","last_clean":"2024-09-14T04:02:51.918439+0000","last_became_active":"2024-09-14T03:57:55.657817+0000","last_became_peered":"2024-09-14T03:57:55.657817+0000","last_unstale":"2024-09-14T04:02:51.918439+0000","last_undegraded":"2024-09-14T04:02:51.918439+0000","last_fullsized":"2024-09-14T04:02:51.918439+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:36.196909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341202+0000","last_change":"2024-09-14T04:02:33.196376+0000","last_active":"2024-09-14T04:02:50.341202+0000","last_peered":"2024-09-14T04:02:50.341202+0000","last_clean":"2024-09-14T04:02:50.341202+0000","last_became_active":"2024-09-14T04:02:33.195804+0000","last_became_peered":"2024-09-14T04:02:33.195804+0000","last_unstale":"2024-09-14T04:02:50.341202+0000","last_undegraded":"2024-09-14T04:02:50.341202+0000","last_fullsized":"2024-09-14T04:02:50.341202+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:58:02.150113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918423+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T04:02:51.918423+0000","last_peered":"2024-09-14T04:02:51.918423+0000","last_clean":"2024-09-14T04:02:51.918423+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T04:02:51.918423+0000","last_undegraded":"2024-09-14T04:02:51.918423+0000","last_fullsized":"2024-09-14T04:02:51.918423+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182266+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:02:33.182266+0000","last_peered":"2024-09-14T04:02:33.182266+0000","last_clean":"2024-09-14T04:02:33.182266+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:02:33.182266+0000","last_undegraded":"2024-09-14T04:02:33.182266+0000","last_fullsized":"2024-09-14T04:02:33.182266+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.598290+0000","last_change":"2024-09-14T04:02:33.189804+0000","last_active":"2024-09-14T04:02:50.598290+0000","last_peered":"2024-09-14T04:02:50.598290+0000","last_clean":"2024-09-14T04:02:50.598290+0000","last_became_active":"2024-09-14T04:02:33.189427+0000","last_became_peered":"2024-09-14T04:02:33.189427+0000","last_unstale":"2024-09-14T04:02:50.598290+0000","last_undegraded":"2024-09-14T04:02:50.598290+0000","last_fullsized":"2024-09-14T04:02:50.598290+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:35.027998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342724+0000","last_change":"2024-09-14T03:57:55.640069+0000","last_active":"2024-09-14T04:02:51.342724+0000","last_peered":"2024-09-14T04:02:51.342724+0000","last_clean":"2024-09-14T04:02:51.342724+0000","last_became_active":"2024-09-14T03:57:55.639653+0000","last_became_peered":"2024-09-14T03:57:55.639653+0000","last_unstale":"2024-09-14T04:02:51.342724+0000","last_undegraded":"2024-09-14T04:02:51.342724+0000","last_fullsized":"2024-09-14T04:02:51.342724+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:16.579752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918366+0000","last_change":"2024-09-14T04:02:33.189648+0000","last_active":"2024-09-14T04:02:51.918366+0000","last_peered":"2024-09-14T04:02:51.918366+0000","last_clean":"2024-09-14T04:02:51.918366+0000","last_became_active":"2024-09-14T04:02:33.188956+0000","last_became_peered":"2024-09-14T04:02:33.188956+0000","last_unstale":"2024-09-14T04:02:51.918366+0000","last_undegraded":"2024-09-14T04:02:51.918366+0000","last_fullsized":"2024-09-14T04:02:51.918366+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:14.851661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.384607+0000","last_change":"2024-09-14T04:02:33.194407+0000","last_active":"2024-09-14T04:02:50.384607+0000","last_peered":"2024-09-14T04:02:50.384607+0000","last_clean":"2024-09-14T04:02:50.384607+0000","last_became_active":"2024-09-14T04:02:33.194049+0000","last_became_peered":"2024-09-14T04:02:33.194049+0000","last_unstale":"2024-09-14T04:02:50.384607+0000","last_undegraded":"2024-09-14T04:02:50.384607+0000","last_fullsized":"2024-09-14T04:02:50.384607+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:09.389524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-09-14T04:02:33.182415+0000","last_change":"2024-09-14T03:57:55.680448+0000","last_active":"2024-09-14T04:02:33.182415+0000","last_peered":"2024-09-14T04:02:33.182415+0000","last_clean":"2024-09-14T04:02:33.182415+0000","last_became_active":"2024-09-14T03:57:55.680199+0000","last_became_peered":"2024-09-14T03:57:55.680199+0000","last_unstale":"2024-09-14T04:02:33.182415+0000","last_undegraded":"2024-09-14T04:02:33.182415+0000","last_fullsized":"2024-09-14T04:02:33.182415+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:03.237323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340655+0000","last_change":"2024-09-14T04:01:59.052174+0000","last_active":"2024-09-14T04:02:51.340655+0000","last_peered":"2024-09-14T04:02:51.340655+0000","last_clean":"2024-09-14T04:02:51.340655+0000","last_became_active":"2024-09-14T04:01:59.051831+0000","last_became_peered":"2024-09-14T04:01:59.051831+0000","last_unstale":"2024-09-14T04:02:51.340655+0000","last_undegraded":"2024-09-14T04:02:51.340655+0000","last_fullsized":"2024-09-14T04:02:51.340655+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:15.224478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341485+0000","last_change":"2024-09-14T04:01:59.055460+0000","last_active":"2024-09-14T04:02:50.341485+0000","last_peered":"2024-09-14T04:02:50.341485+0000","last_clean":"2024-09-14T04:02:50.341485+0000","last_became_active":"2024-09-14T04:01:59.055064+0000","last_became_peered":"2024-09-14T04:01:59.055064+0000","last_unstale":"2024-09-14T04:02:50.341485+0000","last_undegraded":"2024-09-14T04:02:50.341485+0000","last_fullsized":"2024-09-14T04:02:50.341485+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:52.089107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340265+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:02:51.340265+0000","last_peered":"2024-09-14T04:02:51.340265+0000","last_clean":"2024-09-14T04:02:51.340265+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:02:51.340265+0000","last_undegraded":"2024-09-14T04:02:51.340265+0000","last_fullsized":"2024-09-14T04:02:51.340265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917955+0000","last_change":"2024-09-14T04:02:33.194317+0000","last_active":"2024-09-14T04:02:51.917955+0000","last_peered":"2024-09-14T04:02:51.917955+0000","last_clean":"2024-09-14T04:02:51.917955+0000","last_became_active":"2024-09-14T04:02:33.193981+0000","last_became_peered":"2024-09-14T04:02:33.193981+0000","last_unstale":"2024-09-14T04:02:51.917955+0000","last_undegraded":"2024-09-14T04:02:51.917955+0000","last_fullsized":"2024-09-14T04:02:51.917955+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:35:21.725699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343573+0000","last_change":"2024-09-14T04:02:33.192562+0000","last_active":"2024-09-14T04:02:51.343573+0000","last_peered":"2024-09-14T04:02:51.343573+0000","last_clean":"2024-09-14T04:02:51.343573+0000","last_became_active":"2024-09-14T04:02:33.192185+0000","last_became_peered":"2024-09-14T04:02:33.192185+0000","last_unstale":"2024-09-14T04:02:51.343573+0000","last_undegraded":"2024-09-14T04:02:51.343573+0000","last_fullsized":"2024-09-14T04:02:51.343573+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:34:52.201865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":429,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918623+0000","last_change":"2024-09-14T03:57:07.974148+0000","last_active":"2024-09-14T04:02:51.918623+0000","last_peered":"2024-09-14T04:02:51.918623+0000","last_clean":"2024-09-14T04:02:51.918623+0000","last_became_active":"2024-09-14T03:57:07.973612+0000","last_became_peered":"2024-09-14T03:57:07.973612+0000","last_unstale":"2024-09-14T04:02:51.918623+0000","last_undegraded":"2024-09-14T04:02:51.918623+0000","last_fullsized":"2024-09-14T04:02:51.918623+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:13.349866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338527+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:02:50.338527+0000","last_peered":"2024-09-14T04:02:50.338527+0000","last_clean":"2024-09-14T04:02:50.338527+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:02:50.338527+0000","last_undegraded":"2024-09-14T04:02:50.338527+0000","last_fullsized":"2024-09-14T04:02:50.338527+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918082+0000","last_change":"2024-09-14T04:02:33.189844+0000","last_active":"2024-09-14T04:02:51.918082+0000","last_peered":"2024-09-14T04:02:51.918082+0000","last_clean":"2024-09-14T04:02:51.918082+0000","last_became_active":"2024-09-14T04:02:33.189492+0000","last_became_peered":"2024-09-14T04:02:33.189492+0000","last_unstale":"2024-09-14T04:02:51.918082+0000","last_undegraded":"2024-09-14T04:02:51.918082+0000","last_fullsized":"2024-09-14T04:02:51.918082+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:28.814763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625489+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:02:51.625489+0000","last_peered":"2024-09-14T04:02:51.625489+0000","last_clean":"2024-09-14T04:02:51.625489+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:02:51.625489+0000","last_undegraded":"2024-09-14T04:02:51.625489+0000","last_fullsized":"2024-09-14T04:02:51.625489+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918284+0000","last_change":"2024-09-14T03:57:55.681501+0000","last_active":"2024-09-14T04:02:51.918284+0000","last_peered":"2024-09-14T04:02:51.918284+0000","last_clean":"2024-09-14T04:02:51.918284+0000","last_became_active":"2024-09-14T03:57:55.681104+0000","last_became_peered":"2024-09-14T03:57:55.681104+0000","last_unstale":"2024-09-14T04:02:51.918284+0000","last_undegraded":"2024-09-14T04:02:51.918284+0000","last_fullsized":"2024-09-14T04:02:51.918284+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:54.441046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342257+0000","last_change":"2024-09-14T04:02:33.189629+0000","last_active":"2024-09-14T04:02:51.342257+0000","last_peered":"2024-09-14T04:02:51.342257+0000","last_clean":"2024-09-14T04:02:51.342257+0000","last_became_active":"2024-09-14T04:02:33.189317+0000","last_became_peered":"2024-09-14T04:02:33.189317+0000","last_unstale":"2024-09-14T04:02:51.342257+0000","last_undegraded":"2024-09-14T04:02:51.342257+0000","last_fullsized":"2024-09-14T04:02:51.342257+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:09.638142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342665+0000","last_change":"2024-09-14T04:01:59.060194+0000","last_active":"2024-09-14T04:02:51.342665+0000","last_peered":"2024-09-14T04:02:51.342665+0000","last_clean":"2024-09-14T04:02:51.342665+0000","last_became_active":"2024-09-14T04:01:59.059877+0000","last_became_peered":"2024-09-14T04:01:59.059877+0000","last_unstale":"2024-09-14T04:02:51.342665+0000","last_undegraded":"2024-09-14T04:02:51.342665+0000","last_fullsized":"2024-09-14T04:02:51.342665+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:34.061157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918185+0000","last_change":"2024-09-14T04:02:33.186950+0000","last_active":"2024-09-14T04:02:51.918185+0000","last_peered":"2024-09-14T04:02:51.918185+0000","last_clean":"2024-09-14T04:02:51.918185+0000","last_became_active":"2024-09-14T04:02:33.186615+0000","last_became_peered":"2024-09-14T04:02:33.186615+0000","last_unstale":"2024-09-14T04:02:51.918185+0000","last_undegraded":"2024-09-14T04:02:51.918185+0000","last_fullsized":"2024-09-14T04:02:51.918185+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:26.114627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342698+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:02:51.342698+0000","last_peered":"2024-09-14T04:02:51.342698+0000","last_clean":"2024-09-14T04:02:51.342698+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:02:51.342698+0000","last_undegraded":"2024-09-14T04:02:51.342698+0000","last_fullsized":"2024-09-14T04:02:51.342698+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343360+0000","last_change":"2024-09-14T03:57:55.641841+0000","last_active":"2024-09-14T04:02:51.343360+0000","last_peered":"2024-09-14T04:02:51.343360+0000","last_clean":"2024-09-14T04:02:51.343360+0000","last_became_active":"2024-09-14T03:57:55.638515+0000","last_became_peered":"2024-09-14T03:57:55.638515+0000","last_unstale":"2024-09-14T04:02:51.343360+0000","last_undegraded":"2024-09-14T04:02:51.343360+0000","last_fullsized":"2024-09-14T04:02:51.343360+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:07.687766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":113,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338437+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:02:50.338437+0000","last_peered":"2024-09-14T04:02:50.338437+0000","last_clean":"2024-09-14T04:02:50.338437+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:02:50.338437+0000","last_undegraded":"2024-09-14T04:02:50.338437+0000","last_fullsized":"2024-09-14T04:02:50.338437+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340604+0000","last_change":"2024-09-14T03:57:07.985126+0000","last_active":"2024-09-14T04:02:51.340604+0000","last_peered":"2024-09-14T04:02:51.340604+0000","last_clean":"2024-09-14T04:02:51.340604+0000","last_became_active":"2024-09-14T03:57:07.984427+0000","last_became_peered":"2024-09-14T03:57:07.984427+0000","last_unstale":"2024-09-14T04:02:51.340604+0000","last_undegraded":"2024-09-14T04:02:51.340604+0000","last_fullsized":"2024-09-14T04:02:51.340604+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:23.230268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338533+0000","last_change":"2024-09-14T04:02:33.189850+0000","last_active":"2024-09-14T04:02:50.338533+0000","last_peered":"2024-09-14T04:02:50.338533+0000","last_clean":"2024-09-14T04:02:50.338533+0000","last_became_active":"2024-09-14T04:02:33.189444+0000","last_became_peered":"2024-09-14T04:02:33.189444+0000","last_unstale":"2024-09-14T04:02:50.338533+0000","last_undegraded":"2024-09-14T04:02:50.338533+0000","last_fullsized":"2024-09-14T04:02:50.338533+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:23.120760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":119,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340615+0000","last_change":"2024-09-14T04:01:59.052874+0000","last_active":"2024-09-14T04:02:51.340615+0000","last_peered":"2024-09-14T04:02:51.340615+0000","last_clean":"2024-09-14T04:02:51.340615+0000","last_became_active":"2024-09-14T04:01:59.052455+0000","last_became_peered":"2024-09-14T04:01:59.052455+0000","last_unstale":"2024-09-14T04:02:51.340615+0000","last_undegraded":"2024-09-14T04:02:51.340615+0000","last_fullsized":"2024-09-14T04:02:51.340615+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:05.518917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343376+0000","last_change":"2024-09-14T03:57:55.656097+0000","last_active":"2024-09-14T04:02:51.343376+0000","last_peered":"2024-09-14T04:02:51.343376+0000","last_clean":"2024-09-14T04:02:51.343376+0000","last_became_active":"2024-09-14T03:57:55.649947+0000","last_became_peered":"2024-09-14T03:57:55.649947+0000","last_unstale":"2024-09-14T04:02:51.343376+0000","last_undegraded":"2024-09-14T04:02:51.343376+0000","last_fullsized":"2024-09-14T04:02:51.343376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.222621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342967+0000","last_change":"2024-09-14T03:57:55.667306+0000","last_active":"2024-09-14T04:02:51.342967+0000","last_peered":"2024-09-14T04:02:51.342967+0000","last_clean":"2024-09-14T04:02:51.342967+0000","last_became_active":"2024-09-14T03:57:55.661348+0000","last_became_peered":"2024-09-14T03:57:55.661348+0000","last_unstale":"2024-09-14T04:02:51.342967+0000","last_undegraded":"2024-09-14T04:02:51.342967+0000","last_fullsized":"2024-09-14T04:02:51.342967+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:25.408014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343274+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:02:51.343274+0000","last_peered":"2024-09-14T04:02:51.343274+0000","last_clean":"2024-09-14T04:02:51.343274+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:02:51.343274+0000","last_undegraded":"2024-09-14T04:02:51.343274+0000","last_fullsized":"2024-09-14T04:02:51.343274+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342983+0000","last_change":"2024-09-14T03:57:07.974119+0000","last_active":"2024-09-14T04:02:50.342983+0000","last_peered":"2024-09-14T04:02:50.342983+0000","last_clean":"2024-09-14T04:02:50.342983+0000","last_became_active":"2024-09-14T03:57:07.973617+0000","last_became_peered":"2024-09-14T03:57:07.973617+0000","last_unstale":"2024-09-14T04:02:50.342983+0000","last_undegraded":"2024-09-14T04:02:50.342983+0000","last_fullsized":"2024-09-14T04:02:50.342983+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:55.426883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342856+0000","last_change":"2024-09-14T03:57:07.984238+0000","last_active":"2024-09-14T04:02:51.342856+0000","last_peered":"2024-09-14T04:02:51.342856+0000","last_clean":"2024-09-14T04:02:51.342856+0000","last_became_active":"2024-09-14T03:57:07.983677+0000","last_became_peered":"2024-09-14T03:57:07.983677+0000","last_unstale":"2024-09-14T04:02:51.342856+0000","last_undegraded":"2024-09-14T04:02:51.342856+0000","last_fullsized":"2024-09-14T04:02:51.342856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:44:40.642082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918492+0000","last_change":"2024-09-14T04:02:33.190590+0000","last_active":"2024-09-14T04:02:51.918492+0000","last_peered":"2024-09-14T04:02:51.918492+0000","last_clean":"2024-09-14T04:02:51.918492+0000","last_became_active":"2024-09-14T04:02:33.190109+0000","last_became_peered":"2024-09-14T04:02:33.190109+0000","last_unstale":"2024-09-14T04:02:51.918492+0000","last_undegraded":"2024-09-14T04:02:51.918492+0000","last_fullsized":"2024-09-14T04:02:51.918492+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:53:10.464962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343631+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:02:51.343631+0000","last_peered":"2024-09-14T04:02:51.343631+0000","last_clean":"2024-09-14T04:02:51.343631+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:02:51.343631+0000","last_undegraded":"2024-09-14T04:02:51.343631+0000","last_fullsized":"2024-09-14T04:02:51.343631+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.917900+0000","last_change":"2024-09-14T04:01:59.057227+0000","last_active":"2024-09-14T04:02:51.917900+0000","last_peered":"2024-09-14T04:02:51.917900+0000","last_clean":"2024-09-14T04:02:51.917900+0000","last_became_active":"2024-09-14T04:01:59.056799+0000","last_became_peered":"2024-09-14T04:01:59.056799+0000","last_unstale":"2024-09-14T04:02:51.917900+0000","last_undegraded":"2024-09-14T04:02:51.917900+0000","last_fullsized":"2024-09-14T04:02:51.917900+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:54.217497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":102,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342940+0000","last_change":"2024-09-14T03:57:55.649322+0000","last_active":"2024-09-14T04:02:51.342940+0000","last_peered":"2024-09-14T04:02:51.342940+0000","last_clean":"2024-09-14T04:02:51.342940+0000","last_became_active":"2024-09-14T03:57:55.644934+0000","last_became_peered":"2024-09-14T03:57:55.644934+0000","last_unstale":"2024-09-14T04:02:51.342940+0000","last_undegraded":"2024-09-14T04:02:51.342940+0000","last_fullsized":"2024-09-14T04:02:51.342940+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:35.277557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625457+0000","last_change":"2024-09-14T04:01:59.055291+0000","last_active":"2024-09-14T04:02:51.625457+0000","last_peered":"2024-09-14T04:02:51.625457+0000","last_clean":"2024-09-14T04:02:51.625457+0000","last_became_active":"2024-09-14T04:01:59.054919+0000","last_became_peered":"2024-09-14T04:01:59.054919+0000","last_unstale":"2024-09-14T04:02:51.625457+0000","last_undegraded":"2024-09-14T04:02:51.625457+0000","last_fullsized":"2024-09-14T04:02:51.625457+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:22.638368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.340137+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:02:51.340137+0000","last_peered":"2024-09-14T04:02:51.340137+0000","last_clean":"2024-09-14T04:02:51.340137+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:02:51.340137+0000","last_undegraded":"2024-09-14T04:02:51.340137+0000","last_fullsized":"2024-09-14T04:02:51.340137+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341483+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:02:50.341483+0000","last_peered":"2024-09-14T04:02:50.341483+0000","last_clean":"2024-09-14T04:02:50.341483+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:02:50.341483+0000","last_undegraded":"2024-09-14T04:02:50.341483+0000","last_fullsized":"2024-09-14T04:02:50.341483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341694+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:02:50.341694+0000","last_peered":"2024-09-14T04:02:50.341694+0000","last_clean":"2024-09-14T04:02:50.341694+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:02:50.341694+0000","last_undegraded":"2024-09-14T04:02:50.341694+0000","last_fullsized":"2024-09-14T04:02:50.341694+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342812+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:02:51.342812+0000","last_peered":"2024-09-14T04:02:51.342812+0000","last_clean":"2024-09-14T04:02:51.342812+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:02:51.342812+0000","last_undegraded":"2024-09-14T04:02:51.342812+0000","last_fullsized":"2024-09-14T04:02:51.342812+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.342777+0000","last_change":"2024-09-14T04:02:33.192294+0000","last_active":"2024-09-14T04:02:50.342777+0000","last_peered":"2024-09-14T04:02:50.342777+0000","last_clean":"2024-09-14T04:02:50.342777+0000","last_became_active":"2024-09-14T04:02:33.191886+0000","last_became_peered":"2024-09-14T04:02:33.191886+0000","last_unstale":"2024-09-14T04:02:50.342777+0000","last_undegraded":"2024-09-14T04:02:50.342777+0000","last_fullsized":"2024-09-14T04:02:50.342777+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:37:31.901824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338671+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:02:50.338671+0000","last_peered":"2024-09-14T04:02:50.338671+0000","last_clean":"2024-09-14T04:02:50.338671+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:02:50.338671+0000","last_undegraded":"2024-09-14T04:02:50.338671+0000","last_fullsized":"2024-09-14T04:02:50.338671+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918187+0000","last_change":"2024-09-14T04:02:33.187805+0000","last_active":"2024-09-14T04:02:51.918187+0000","last_peered":"2024-09-14T04:02:51.918187+0000","last_clean":"2024-09-14T04:02:51.918187+0000","last_became_active":"2024-09-14T04:02:33.187425+0000","last_became_peered":"2024-09-14T04:02:33.187425+0000","last_unstale":"2024-09-14T04:02:51.918187+0000","last_undegraded":"2024-09-14T04:02:51.918187+0000","last_fullsized":"2024-09-14T04:02:51.918187+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:42:32.471118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343272+0000","last_change":"2024-09-14T04:02:33.197914+0000","last_active":"2024-09-14T04:02:51.343272+0000","last_peered":"2024-09-14T04:02:51.343272+0000","last_clean":"2024-09-14T04:02:51.343272+0000","last_became_active":"2024-09-14T04:02:33.197447+0000","last_became_peered":"2024-09-14T04:02:33.197447+0000","last_unstale":"2024-09-14T04:02:51.343272+0000","last_undegraded":"2024-09-14T04:02:51.343272+0000","last_fullsized":"2024-09-14T04:02:51.343272+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:08:47.196932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918063+0000","last_change":"2024-09-14T04:02:33.196157+0000","last_active":"2024-09-14T04:02:51.918063+0000","last_peered":"2024-09-14T04:02:51.918063+0000","last_clean":"2024-09-14T04:02:51.918063+0000","last_became_active":"2024-09-14T04:02:33.195682+0000","last_became_peered":"2024-09-14T04:02:33.195682+0000","last_unstale":"2024-09-14T04:02:51.918063+0000","last_undegraded":"2024-09-14T04:02:51.918063+0000","last_fullsized":"2024-09-14T04:02:51.918063+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:04.357130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.625500+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:02:51.625500+0000","last_peered":"2024-09-14T04:02:51.625500+0000","last_clean":"2024-09-14T04:02:51.625500+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:02:51.625500+0000","last_undegraded":"2024-09-14T04:02:51.625500+0000","last_fullsized":"2024-09-14T04:02:51.625500+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918026+0000","last_change":"2024-09-14T04:02:33.195513+0000","last_active":"2024-09-14T04:02:51.918026+0000","last_peered":"2024-09-14T04:02:51.918026+0000","last_clean":"2024-09-14T04:02:51.918026+0000","last_became_active":"2024-09-14T04:02:33.195109+0000","last_became_peered":"2024-09-14T04:02:33.195109+0000","last_unstale":"2024-09-14T04:02:51.918026+0000","last_undegraded":"2024-09-14T04:02:51.918026+0000","last_fullsized":"2024-09-14T04:02:51.918026+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:30.591898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342747+0000","last_change":"2024-09-14T04:02:33.198166+0000","last_active":"2024-09-14T04:02:51.342747+0000","last_peered":"2024-09-14T04:02:51.342747+0000","last_clean":"2024-09-14T04:02:51.342747+0000","last_became_active":"2024-09-14T04:02:33.197287+0000","last_became_peered":"2024-09-14T04:02:33.197287+0000","last_unstale":"2024-09-14T04:02:51.342747+0000","last_undegraded":"2024-09-14T04:02:51.342747+0000","last_fullsized":"2024-09-14T04:02:51.342747+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:41:48.137435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342622+0000","last_change":"2024-09-14T04:02:33.192061+0000","last_active":"2024-09-14T04:02:51.342622+0000","last_peered":"2024-09-14T04:02:51.342622+0000","last_clean":"2024-09-14T04:02:51.342622+0000","last_became_active":"2024-09-14T04:02:33.191529+0000","last_became_peered":"2024-09-14T04:02:33.191529+0000","last_unstale":"2024-09-14T04:02:51.342622+0000","last_undegraded":"2024-09-14T04:02:51.342622+0000","last_fullsized":"2024-09-14T04:02:51.342622+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:06.031084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918787+0000","last_change":"2024-09-14T04:02:33.191433+0000","last_active":"2024-09-14T04:02:51.918787+0000","last_peered":"2024-09-14T04:02:51.918787+0000","last_clean":"2024-09-14T04:02:51.918787+0000","last_became_active":"2024-09-14T04:02:33.190554+0000","last_became_peered":"2024-09-14T04:02:33.190554+0000","last_unstale":"2024-09-14T04:02:51.918787+0000","last_undegraded":"2024-09-14T04:02:51.918787+0000","last_fullsized":"2024-09-14T04:02:51.918787+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:58.975784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342433+0000","last_change":"2024-09-14T04:01:59.053348+0000","last_active":"2024-09-14T04:02:51.342433+0000","last_peered":"2024-09-14T04:02:51.342433+0000","last_clean":"2024-09-14T04:02:51.342433+0000","last_became_active":"2024-09-14T04:01:59.053004+0000","last_became_peered":"2024-09-14T04:01:59.053004+0000","last_unstale":"2024-09-14T04:02:51.342433+0000","last_undegraded":"2024-09-14T04:02:51.342433+0000","last_fullsized":"2024-09-14T04:02:51.342433+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:56:28.268868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918249+0000","last_change":"2024-09-14T04:02:33.191341+0000","last_active":"2024-09-14T04:02:51.918249+0000","last_peered":"2024-09-14T04:02:51.918249+0000","last_clean":"2024-09-14T04:02:51.918249+0000","last_became_active":"2024-09-14T04:02:33.190081+0000","last_became_peered":"2024-09-14T04:02:33.190081+0000","last_unstale":"2024-09-14T04:02:51.918249+0000","last_undegraded":"2024-09-14T04:02:51.918249+0000","last_fullsized":"2024-09-14T04:02:51.918249+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:35:52.740363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":98,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.342908+0000","last_change":"2024-09-14T04:01:59.057068+0000","last_active":"2024-09-14T04:02:51.342908+0000","last_peered":"2024-09-14T04:02:51.342908+0000","last_clean":"2024-09-14T04:02:51.342908+0000","last_became_active":"2024-09-14T04:01:59.056702+0000","last_became_peered":"2024-09-14T04:01:59.056702+0000","last_unstale":"2024-09-14T04:02:51.342908+0000","last_undegraded":"2024-09-14T04:02:51.342908+0000","last_fullsized":"2024-09-14T04:02:51.342908+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:23.708120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341445+0000","last_change":"2024-09-14T04:02:33.192285+0000","last_active":"2024-09-14T04:02:50.341445+0000","last_peered":"2024-09-14T04:02:50.341445+0000","last_clean":"2024-09-14T04:02:50.341445+0000","last_became_active":"2024-09-14T04:02:33.191832+0000","last_became_peered":"2024-09-14T04:02:33.191832+0000","last_unstale":"2024-09-14T04:02:50.341445+0000","last_undegraded":"2024-09-14T04:02:50.341445+0000","last_fullsized":"2024-09-14T04:02:50.341445+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:38:16.739279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":136,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.346840+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:02:51.346840+0000","last_peered":"2024-09-14T04:02:51.346840+0000","last_clean":"2024-09-14T04:02:51.346840+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:02:51.346840+0000","last_undegraded":"2024-09-14T04:02:51.346840+0000","last_fullsized":"2024-09-14T04:02:51.346840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341322+0000","last_change":"2024-09-14T04:02:33.194384+0000","last_active":"2024-09-14T04:02:50.341322+0000","last_peered":"2024-09-14T04:02:50.341322+0000","last_clean":"2024-09-14T04:02:50.341322+0000","last_became_active":"2024-09-14T04:02:33.194027+0000","last_became_peered":"2024-09-14T04:02:33.194027+0000","last_unstale":"2024-09-14T04:02:50.341322+0000","last_undegraded":"2024-09-14T04:02:50.341322+0000","last_fullsized":"2024-09-14T04:02:50.341322+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:46:29.339591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.338808+0000","last_change":"2024-09-14T03:57:08.990606+0000","last_active":"2024-09-14T04:02:50.338808+0000","last_peered":"2024-09-14T04:02:50.338808+0000","last_clean":"2024-09-14T04:02:50.338808+0000","last_became_active":"2024-09-14T03:57:08.979113+0000","last_became_peered":"2024-09-14T03:57:08.979113+0000","last_unstale":"2024-09-14T04:02:50.338808+0000","last_undegraded":"2024-09-14T04:02:50.338808+0000","last_fullsized":"2024-09-14T04:02:50.338808+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:53.542287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918089+0000","last_change":"2024-09-14T04:02:33.194588+0000","last_active":"2024-09-14T04:02:51.918089+0000","last_peered":"2024-09-14T04:02:51.918089+0000","last_clean":"2024-09-14T04:02:51.918089+0000","last_became_active":"2024-09-14T04:02:33.193732+0000","last_became_peered":"2024-09-14T04:02:33.193732+0000","last_unstale":"2024-09-14T04:02:51.918089+0000","last_undegraded":"2024-09-14T04:02:51.918089+0000","last_fullsized":"2024-09-14T04:02:51.918089+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:50.529572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":109,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.918743+0000","last_change":"2024-09-14T04:02:33.193693+0000","last_active":"2024-09-14T04:02:51.918743+0000","last_peered":"2024-09-14T04:02:51.918743+0000","last_clean":"2024-09-14T04:02:51.918743+0000","last_became_active":"2024-09-14T04:02:33.192884+0000","last_became_peered":"2024-09-14T04:02:33.192884+0000","last_unstale":"2024-09-14T04:02:51.918743+0000","last_undegraded":"2024-09-14T04:02:51.918743+0000","last_fullsized":"2024-09-14T04:02:51.918743+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:21.242232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-09-14T04:02:50.341118+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:02:50.341118+0000","last_peered":"2024-09-14T04:02:50.341118+0000","last_clean":"2024-09-14T04:02:50.341118+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:02:50.341118+0000","last_undegraded":"2024-09-14T04:02:50.341118+0000","last_fullsized":"2024-09-14T04:02:50.341118+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":102,"reported_epoch":133,"state":"active+clean","last_fresh":"2024-09-14T04:02:51.343134+0000","last_change":"2024-09-14T04:01:59.054141+0000","last_active":"2024-09-14T04:02:51.343134+0000","last_peered":"2024-09-14T04:02:51.343134+0000","last_clean":"2024-09-14T04:02:51.343134+0000","last_became_active":"2024-09-14T04:01:59.053397+0000","last_became_peered":"2024-09-14T04:01:59.053397+0000","last_unstale":"2024-09-14T04:02:51.343134+0000","last_undegraded":"2024-09-14T04:02:51.343134+0000","last_fullsized":"2024-09-14T04:02:51.343134+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:38.323001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13714,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201556,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466452,"kb_used_data":2184,"kb_used_omap":1,"kb_used_meta":464254,"kb_avail":93278700,"statfs":{"total":95995035648,"available":95517388800,"internally_reserved":0,"allocated":2236416,"data_stored":1440220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":475396555},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463194,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416392,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":93328760,"statfs":{"total":95995035648,"available":95568650240,"internally_reserved":0,"allocated":2224128,"data_stored":1439573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":424147399},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724830,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640580,"kb_used_data":1588,"kb_used_omap":1,"kb_used_meta":638974,"kb_avail":93104572,"statfs":{"total":95995035648,"available":95339081728,"internally_reserved":0,"allocated":1626112,"data_stored":845304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":654309830},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986466,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560208,"kb_used_data":1600,"kb_used_omap":1,"kb_used_meta":558590,"kb_avail":93184944,"statfs":{"total":95995035648,"available":95421382656,"internally_reserved":0,"allocated":1638400,"data_stored":849543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":571996614},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748495,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84988,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":83387,"kb_avail":93660164,"statfs":{"total":95995035648,"available":95908007936,"internally_reserved":0,"allocated":1622016,"data_stored":845016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":85388844},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912022,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76036,"kb_used_data":1588,"kb_used_omap":6,"kb_used_meta":74425,"kb_avail":93669116,"statfs":{"total":95995035648,"available":95917174784,"internally_reserved":0,"allocated":1626112,"data_stored":845542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":76211769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468038,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190392,"kb_used_data":1580,"kb_used_omap":5,"kb_used_meta":188794,"kb_avail":93554760,"statfs":{"total":95995035648,"available":95800074240,"internally_reserved":0,"allocated":1617920,"data_stored":843525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":193325407},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":102,"seq":438086664272,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156872,"kb_used_data":2172,"kb_used_omap":5,"kb_used_meta":154682,"kb_avail":93588280,"statfs":{"total":95995035648,"available":95834398720,"internally_reserved":0,"allocated":2224128,"data_stored":1435932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5976,"internal_metadata":158394536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4382,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:03:39.668 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T04:03:39.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-14T04:03:39.877 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:41.353 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:41 smithi001 ceph-mon[107399]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:41.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:41 smithi001 ceph-mon[112165]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:41 smithi012 ceph-mon[101180]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:41.856 INFO:teuthology.orchestra.run.smithi001.stdout:epoch 133 2024-09-14T04:03:41.856 INFO:teuthology.orchestra.run.smithi001.stdout:fsid 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T04:03:41.856 INFO:teuthology.orchestra.run.smithi001.stdout:created 2024-09-14T03:35:50.494718+0000 2024-09-14T04:03:41.856 INFO:teuthology.orchestra.run.smithi001.stdout:modified 2024-09-14T04:02:51.336790+0000 2024-09-14T04:03:41.856 INFO:teuthology.orchestra.run.smithi001.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:crush_version 22 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:full_ratio 0.95 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:backfillfull_ratio 0.9 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:nearfull_ratio 0.85 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:require_min_compat_client luminous 2024-09-14T04:03:41.857 INFO:teuthology.orchestra.run.smithi001.stdout:min_compat_client luminous 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:require_osd_release reef 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:stretch_mode_enabled false 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 66 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.00 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 72 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:max_osd 8 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 up in weight 1 up_from 102 up_thru 130 down_at 92 last_clean_interval [8,91) [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157] [v2:172.21.15.1:6804/1278848157,v1:172.21.15.1:6805/1278848157] exists,up 86fadbf5-5c8f-4719-b817-f0ff28fb4caa 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 up in weight 1 up_from 108 up_thru 130 down_at 105 last_clean_interval [13,104) [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422] [v2:172.21.15.1:6812/2795716422,v1:172.21.15.1:6813/2795716422] exists,up cbf0be8f-6a29-4e63-8dca-4d830fcd7d47 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 up in weight 1 up_from 125 up_thru 130 down_at 122 last_clean_interval [18,121) [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272] [v2:172.21.15.1:6820/4111579272,v1:172.21.15.1:6821/4111579272] exists,up ca27ddd4-8678-4b03-be41-f1600e007add 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 up in weight 1 up_from 130 up_thru 130 down_at 127 last_clean_interval [26,126) [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471] [v2:172.21.15.1:6828/3997365471,v1:172.21.15.1:6829/3997365471] exists,up 7004b178-c338-490e-a6f2-fb8510c93c29 2024-09-14T04:03:41.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 up in weight 1 up_from 31 up_thru 130 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415] [v2:172.21.15.12:6802/2859222415,v1:172.21.15.12:6803/2859222415] exists,up 40284648-ea27-4db8-977d-cfd9e41c3b7f 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 up in weight 1 up_from 39 up_thru 132 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883] [v2:172.21.15.12:6810/2995357883,v1:172.21.15.12:6811/2995357883] exists,up ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6 up in weight 1 up_from 47 up_thru 127 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370] [v2:172.21.15.12:6818/2651831370,v1:172.21.15.12:6819/2651831370] exists,up 786e0de6-b9fc-4b30-bd1f-a888694aeeab 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 up in weight 1 up_from 55 up_thru 130 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999] [v2:172.21.15.12:6826/1667024999,v1:172.21.15.12:6827/1667024999] exists,up 409b8e01-c0ad-4f5c-b67a-95b914426b8a 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.5 [3,0] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.a [4,0] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.13 [7,0] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.1a [4,0] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.1b [4,2] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 3.1d [4,0] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.0 [3,2] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.4 [1,5] 2024-09-14T04:03:41.859 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.13 [1,2] 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.14 [1,2] 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.17 [3,0] 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:pg_upmap_items 4.1d [1,5] 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6800/3323570302 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/1739684728 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2483217146 expires 2024-09-15T03:36:16.593660+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6801/2953284694 expires 2024-09-15T03:36:16.593660+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/4291653236 expires 2024-09-15T03:36:40.470365+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6801/226348322 expires 2024-09-15T03:36:40.470365+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/1932731734 expires 2024-09-15T03:36:40.470365+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/2389061674 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.860 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6800/1008160518 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:6834/3701332877 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/4171009491 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2949027530 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/739412744 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6800/2630182969 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2817796360 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6800/2953284694 expires 2024-09-15T03:36:16.593660+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6801/1008160518 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6800/226348322 expires 2024-09-15T03:36:40.470365+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/3177706589 expires 2024-09-15T03:36:16.593660+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/1135126809 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/2685165922 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/4146840439 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.861 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/41633275 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2862372200 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/1621027980 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/416113000 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6801/3323570302 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/3674234575 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2665058722 expires 2024-09-15T03:36:16.593660+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2358137785 expires 2024-09-15T03:36:40.470365+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:6835/3701332877 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/3915177672 expires 2024-09-15T03:41:16.654706+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:0/2249577072 expires 2024-09-15T03:50:48.967020+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/3549839308 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.1:6801/2630182969 expires 2024-09-15T03:46:31.869280+0000 2024-09-14T04:03:41.862 INFO:teuthology.orchestra.run.smithi001.stdout:blocklist 172.21.15.12:0/3359585252 expires 2024-09-15T03:47:16.144756+0000 2024-09-14T04:03:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/13046638' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:03:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/13046638' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:03:42.563 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 102 up_thru 130 down_at 92 last_clean_interval [8,91) [v2:172.21.15.1:6802/1278848157,v1:172.21.15.1:6803/1278848157] [v2:172.21.15.1:6804/1278848157,v1:172.21.15.1:6805/1278848157] exists,up 86fadbf5-5c8f-4719-b817-f0ff28fb4caa', 'osd.1 up in weight 1 up_from 108 up_thru 130 down_at 105 last_clean_interval [13,104) [v2:172.21.15.1:6810/2795716422,v1:172.21.15.1:6811/2795716422] [v2:172.21.15.1:6812/2795716422,v1:172.21.15.1:6813/2795716422] exists,up cbf0be8f-6a29-4e63-8dca-4d830fcd7d47', 'osd.2 up in weight 1 up_from 125 up_thru 130 down_at 122 last_clean_interval [18,121) [v2:172.21.15.1:6818/4111579272,v1:172.21.15.1:6819/4111579272] [v2:172.21.15.1:6820/4111579272,v1:172.21.15.1:6821/4111579272] exists,up ca27ddd4-8678-4b03-be41-f1600e007add', 'osd.3 up in weight 1 up_from 130 up_thru 130 down_at 127 last_clean_interval [26,126) [v2:172.21.15.1:6826/3997365471,v1:172.21.15.1:6827/3997365471] [v2:172.21.15.1:6828/3997365471,v1:172.21.15.1:6829/3997365471] exists,up 7004b178-c338-490e-a6f2-fb8510c93c29', 'osd.4 up in weight 1 up_from 31 up_thru 130 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6800/2859222415,v1:172.21.15.12:6801/2859222415] [v2:172.21.15.12:6802/2859222415,v1:172.21.15.12:6803/2859222415] exists,up 40284648-ea27-4db8-977d-cfd9e41c3b7f', 'osd.5 up in weight 1 up_from 39 up_thru 132 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6808/2995357883,v1:172.21.15.12:6809/2995357883] [v2:172.21.15.12:6810/2995357883,v1:172.21.15.12:6811/2995357883] exists,up ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6', 'osd.6 up in weight 1 up_from 47 up_thru 127 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6816/2651831370,v1:172.21.15.12:6817/2651831370] [v2:172.21.15.12:6818/2651831370,v1:172.21.15.12:6819/2651831370] exists,up 786e0de6-b9fc-4b30-bd1f-a888694aeeab', 'osd.7 up in weight 1 up_from 55 up_thru 130 down_at 0 last_clean_interval [0,0) [v2:172.21.15.12:6824/1667024999,v1:172.21.15.12:6825/1667024999] [v2:172.21.15.12:6826/1667024999,v1:172.21.15.12:6827/1667024999] exists,up 409b8e01-c0ad-4f5c-b67a-95b914426b8a'] 2024-09-14T04:03:42.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-14T04:03:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/13046638' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:03:42.778 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:43 smithi001 ceph-mon[112165]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:43 smithi001 ceph-mon[107399]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:43 smithi012 ceph-mon[101180]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "name": "mon.a", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "cluster": "ceph", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "admin_socket_mode": "", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_client_required": "cephx, none", 2024-09-14T04:03:44.489 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_cluster_required": "cephx", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_debug": "true", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_service_required": "cephx", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "auth_supported": "", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_aio": "true", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_aio_poll_ms": "250", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_aio_reap_max": "16", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_async_discard_threads": "0", 2024-09-14T04:03:44.490 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_block_size": "4096", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_debug_aio": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_debug_inflight_ios": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_enable_discard": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_flock_retry": "3", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_inject_crash": "0", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_ioring": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_ioring_hipri": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-14T04:03:44.491 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bdev_type": "", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_alloc_size": "1048576", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_allocator": "hybrid", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_buffered_io": "true", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_check_for_zeros": "false", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-14T04:03:44.492 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_compact_log_sync": "false", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_max_log_runway": "4194304", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_max_prefetch": "1048576", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_min_flush_size": "524288", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_min_log_runway": "1048576", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_replay_recovery": "false", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluefs_sync_write": "false", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-14T04:03:44.493 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_allocation_from_file": "true", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_allocator": "hybrid", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_async_db_compaction": "true", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bdev_label_multi": "true", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-14T04:03:44.494 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_create": "true", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_db_create": "false", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_db_path": "", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_db_size": "0", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_path": "", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_preallocate_file": "false", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_size": "107374182400", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_wal_create": "false", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_wal_path": "", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_block_wal_size": "100663296", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bluefs": "true", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-14T04:03:44.495 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_autotune": "true", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_size": "0", 2024-09-14T04:03:44.496 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cache_type": "2q", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_clone_cow": "true", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-14T04:03:44.497 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_mode": "none", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_csum_type": "crc32c", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_freelist": "false", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-14T04:03:44.498 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_small_allocations": "0", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_default_buffered_read": "true", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_default_buffered_write": "false", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-14T04:03:44.499 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fail_eio": "false", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-14T04:03:44.500 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_mount": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_umount": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_ignore_data_csum": "false", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-14T04:03:44.501 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_log_op_age": "5.000000", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_alloc_size": "0", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_blob_size": "0", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_max_deferred_txc": "32", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_min_alloc_size": "0", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-14T04:03:44.502 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_nid_prealloc": "1024", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_retry_disk_reads": "3", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_rocksdb_cf": "true", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-14T04:03:44.503 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_spdk_mem": "512", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_tracing": "false", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-14T04:03:44.504 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_write_v2": "false", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_write_v2_random": "false", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "bluestore_zero_block_detection": "false", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-14T04:03:44.505 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_cluster_require_signatures": "false", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_cluster_require_version": "2", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_require_signatures": "false", 2024-09-14T04:03:44.506 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_require_version": "2", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_service_require_signatures": "false", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_service_require_version": "2", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "cephx_sign_messages": "true", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "chdir": "", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_acl_type": "", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_asio_thread_count": "2", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_cache_mid": "0.750000", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_cache_size": "16384", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_caps_release_delay": "5", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_check_pool_perm": "true", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_debug_force_sync_read": "false", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_debug_getattr_caps": "false", 2024-09-14T04:03:44.507 INFO:teuthology.orchestra.run.smithi001.stdout: "client_debug_inject_features": "", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_debug_inject_tick_delay": "0", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_die_on_failed_remount": "false", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_dirsize_rbytes": "true", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_force_lazyio": "false", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_fs": "", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_inject_release_failure": "false", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_max_inline_size": "4096", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_mds_namespace": "", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_metadata": "", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_mount_gid": "-1", 2024-09-14T04:03:44.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_mount_timeout": "300", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_mount_uid": "-1", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_mountpoint": "/", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_notify_timeout": "10", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc": "true", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc_max_dirty": "104857600", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc_max_objects": "1000", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc_size": "209715200", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_oc_target_dirty": "8388608", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_permissions": "true", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_quota": "true", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_quota_df": "true", 2024-09-14T04:03:44.509 INFO:teuthology.orchestra.run.smithi001.stdout: "client_readahead_max_bytes": "0", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_readahead_max_periods": "4", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_readahead_min": "131072", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_reconnect_stale": "false", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_shutdown_timeout": "30", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_snapdir": ".snap", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_tick_interval": "1", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_trace": "", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_try_dentry_invalidate": "false", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_use_faked_inos": "false", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_use_random_mds": "false", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_graylog": "false", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_graylog_port": "12201", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_monitors": "default=true", 2024-09-14T04:03:44.510 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_syslog": "false", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "clog_to_syslog_level": "info", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "cluster_addr": "-", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "cluster_network": "", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "cluster_network_interface": "", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "compressor_zlib_isal": "false", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "compressor_zlib_level": "5", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "compressor_zlib_winsize": "-15", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "compressor_zstd_level": "1", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_alien_op_num_threads": "6", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-14T04:03:44.511 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_osd_stat_interval": "0", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_seastar_cpu_cores": "", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crimson_seastar_num_threads": "0", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crush_location": "", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crush_location_hook": "", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "crush_location_hook_timeout": "10", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "daemonize": "false", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "daos_pool": "tank", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-14T04:03:44.512 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_alienstore": "0/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_allow_any_pool_priority": "false", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_asok": "1/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_asok_assert_abort": "false", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_asserts_on_shutdown": "true", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_auth": "1/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_bdev": "1/3", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_bluefs": "1/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_bluestore": "1/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_buffer": "0/1", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_ceph_dedup": "0/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_ceph_exporter": "1/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_cephfs_mirror": "0/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_cephsqlite": "0/5", 2024-09-14T04:03:44.513 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_client": "0/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_compressor": "1/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_context": "0/1", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_crimson_interrupt": "0/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_crush": "1/1", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_crypto": "1/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_cyanstore": "0/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_deliberately_leak_memory": "false", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_disable_randomized_ping": "false", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_dpdk": "1/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_eventtrace": "1/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_filer": "0/1", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_filestore": "1/3", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_finisher": "1/1", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_fuse": "1/5", 2024-09-14T04:03:44.514 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_heartbeat_testing_span": "0", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_heartbeatmap": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_javaclient": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_journal": "1/3", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_journaler": "0/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_kstore": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_lockdep": "0/1", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mclock": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_balancer": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_locker": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_log": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_log_expire": "1/5", 2024-09-14T04:03:44.515 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_migrator": "1/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mds_quiesce": "3/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_memstore": "1/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mgr": "2/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mgrc": "1/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_mon": "20/20", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_monc": "0/10", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_ms": "1/1", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_none": "0/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_objclass": "0/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_objectcacher": "0/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_objecter": "0/1", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_optracker": "0/5", 2024-09-14T04:03:44.516 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_osd": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_paxos": "20/20", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_perfcounter": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_prioritycache": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rados": "0/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rbd": "0/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rbd_mirror": "0/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rbd_pwl": "0/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rbd_replay": "0/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_refs": "0/0", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_reserver": "1/1", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_access": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_datacache": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_dbstore": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_flight": "1/5", 2024-09-14T04:03:44.517 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-14T04:03:44.518 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_notification": "1/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rgw_sync": "1/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_rocksdb": "4/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_backref": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_cache": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_cleaner": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_device": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_epm": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_journal": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_lba": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_odata": "0/5", 2024-09-14T04:03:44.519 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_omap": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_onode": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_t": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_seastore_tm": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_striper": "0/1", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_test": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_throttle": "1/1", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_timer": "0/1", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_tp": "0/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "debug_trace": "1/5", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "device_failure_prediction_mode": "none", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-14T04:03:44.520 INFO:teuthology.orchestra.run.smithi001.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "err_to_graylog": "false", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "err_to_journald": "false", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "err_to_stderr": "true", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "err_to_syslog": "false", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event_tracing": "false", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_addr": "0.0.0.0", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_cert_file": "", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_http_port": "9926", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_key_file": "", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_prio_limit": "5", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_sort_metrics": "true", 2024-09-14T04:03:44.521 INFO:teuthology.orchestra.run.smithi001.stdout: "exporter_stats_period": "5", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "fake_statfs_for_testing": "0", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "fatal_signal_handlers": "true", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filer_max_purge_ops": "10", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filer_max_truncate_ops": "128", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_apply_finisher_threads": "1", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_blackhole": "false", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_btrfs_clone_range": "true", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_btrfs_snap": "true", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_caller_concurrency": "10", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_collect_device_partition_information": "true", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_commit_timeout": "600.000000", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_debug_inject_read_err": "false", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_debug_omap_check": "false", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_debug_verify_split": "false", 2024-09-14T04:03:44.522 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_dump_file": "", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fadvise": "true", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fail_eio": "true", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fd_cache_shards": "16", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fd_cache_size": "128", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fiemap": "false", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fiemap_threshold": "4096", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_inject_stall": "0", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_journal_parallel": "false", 2024-09-14T04:03:44.523 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_journal_trailing": "false", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_journal_writeahead": "false", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_kill_at": "0", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattrs": "0", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-14T04:03:44.524 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_xattr_value_size": "0", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_merge_threshold": "-10", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_odsync_write": "false", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_omap_backend": "rocksdb", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_omap_backend_path": "", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-14T04:03:44.525 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_op_thread_timeout": "60", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_op_threads": "2", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_punch_hole": "false", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_queue_max_ops": "50", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_seek_data_hole": "false", 2024-09-14T04:03:44.526 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_sloppy_crc": "false", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_splice": "false", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_split_multiple": "2", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_split_rand_factor": "20", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_update_to": "1000", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_enable": "true", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-14T04:03:44.527 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_xfs_extsize": "false", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "filestore_zfs_snap": "false", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fio_dir": "/tmp/fio", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fsid": "5626d550-724a-11ef-bceb-c7b262605968", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_allow_other": "true", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_atomic_o_trunc": "true", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_big_writes": "true", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_debug": "false", 2024-09-14T04:03:44.528 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_default_permissions": "false", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_disable_pagecache": "false", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_max_write": "0", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_multithreaded": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_require_active_mds": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_set_user_groups": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_splice_move": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_splice_read": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_splice_write": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "fuse_use_invalidate_cb": "true", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "gss_target_name": "ceph", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "heartbeat_file": "", 2024-09-14T04:03:44.529 INFO:teuthology.orchestra.run.smithi001.stdout: "heartbeat_inject_failure": "0", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "heartbeat_interval": "5", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "host": "smithi001", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-14T04:03:44.530 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "inject_early_sigterm": "false", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "jaeger_agent_port": "6799", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "jaeger_tracing_enable": "false", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_aio": "true", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_align_min_size": "65536", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_block_align": "true", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_block_size": "4096", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_dio": "true", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_discard": "false", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_force_aio": "false", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_ignore_corruption": "false", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_max_write_bytes": "10485760", 2024-09-14T04:03:44.531 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_max_write_entries": "100", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_replay_from": "0", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_write_header_frequency": "0", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journal_zero_on_create": "false", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journaler_prefetch_periods": "10", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journaler_prezero_periods": "5", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "journaler_write_head_interval": "15", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "key": "", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "keyfile": "", 2024-09-14T04:03:44.532 INFO:teuthology.orchestra.run.smithi001.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_backend": "rocksdb", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_default_stripe_size": "65536", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_fsck_on_mount": "false", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_max_bytes": "67108864", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_max_ops": "512", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_nid_prealloc": "1024", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_onode_map_size": "1024", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_sync_submit_transaction": "false", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "kstore_sync_transaction": "false", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "librados_thread_count": "2", 2024-09-14T04:03:44.533 INFO:teuthology.orchestra.run.smithi001.stdout: "lockdep": "false", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "lockdep_force_backtrace": "false", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_coarse_timestamps": "true", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_flush_on_exit": "false", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_graylog_host": "127.0.0.1", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_graylog_port": "12201", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_max_new": "1000", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_max_recent": "10000", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_stderr_prefix": "", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_stop_at_utilization": "0.970000", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_to_file": "true", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_to_graylog": "false", 2024-09-14T04:03:44.534 INFO:teuthology.orchestra.run.smithi001.stdout: "log_to_journald": "false", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "log_to_stderr": "false", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "log_to_syslog": "false", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "max_rotating_auth_attempts": "10", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_action_on_write_error": "1", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_alternate_name_max": "8192", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_asio_thread_count": "2", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_export_pin": "true", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_fragment_dirs": "true", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_fragment_interval": "5", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-14T04:03:44.535 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_interval": "10", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_max": "-1", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_max_until": "-1", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_merge_size": "50", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_midchunk": "0.300000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_min_start": "0.200000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_minchunk": "0.001000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_mode": "0", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_need_max": "1.200000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_need_min": "0.800000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_overload_epochs": "2", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-14T04:03:44.536 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_split_bits": "3", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_split_size": "10000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_target_decay": "10.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_beacon_grace": "15.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_beacon_interval": "4.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_mid": "0.700000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_quiesce_delay": "0", 2024-09-14T04:03:44.537 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_release_free_interval": "10", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_reservation": "0.050000", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_trim_interval": "1", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cache_trim_threshold": "262144", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_client_prealloc_inos": "1000", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-14T04:03:44.538 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_connect_bootstrapping": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_damage_table_max_entries": "10000", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_debug_auth_pins": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_debug_frag": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_debug_scatterstat": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_debug_subtrees": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_decay_halflife": "5.000000", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_default_dir_hash": "2", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_defer_session_stale": "true", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_deny_all_reconnect": "false", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dir_keys_per_op": "16384", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dir_max_commit_size": "10", 2024-09-14T04:03:44.539 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dir_max_entries": "0", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dir_prefetch": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dump_cache_on_map": "false", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_early_reply": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_enable_op_tracker": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_enforce_unique_name": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_export_ephemeral_random": "true", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-14T04:03:44.540 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_health_summarize_threshold": "10", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_health_dummy": "false", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_migrator_session_race": "false", 2024-09-14T04:03:44.541 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_join_fs": "", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_journal_format": "1", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_create_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_dirfrag_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_export_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_import_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_journal_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_journal_expire_at": "0", 2024-09-14T04:03:44.542 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_journal_replay_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_link_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_mdstable_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_openc_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_rename_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_kill_shutdown_at": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_event_large_threshold": "524288", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_events_per_segment": "1024", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_max_events": "-1", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_max_segments": "128", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_pause": "false", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_segment_size": "0", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-14T04:03:44.543 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_trim_threshold": "128", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_log_warn_factor": "2.000000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_caps_per_client": "1048576", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_completed_flushes": "100000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_completed_requests": "100000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_export_size": "20971520", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_file_recover": "32", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_purge_files": "64", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_purge_ops": "8192", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-14T04:03:44.544 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_max_snaps_per_dir": "100", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_metrics_update_interval": "2", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_min_caps_per_client": "100", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_min_caps_working_set": "10000", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_numa_node": "-1", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_complaint_time": "30.000000", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_history_duration": "600", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_history_size": "20", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_history_slow_op_size": "20", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_op_log_threshold": "5", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_ping_grace": "15", 2024-09-14T04:03:44.545 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_ping_interval": "5", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_max_caps": "30000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_recall_warning_threshold": "262144", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_replay_interval": "1.000000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-14T04:03:44.546 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_root_ino_gid": "0", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_root_ino_uid": "0", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_scrub_stats_review_period": "1", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-14T04:03:44.547 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_shutdown_check": "0", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_skip_ino": "0", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_snap_max_uid": "4294967294", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_snap_min_uid": "0", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_snap_rstat": "false", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_standby_replay_damaged": "false", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_symlink_recovery": "true", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_thrash_exports": "0", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_thrash_fragments": "0", 2024-09-14T04:03:44.548 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_tick_interval": "5.000000", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_valgrind_exit": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_verify_backtrace": "1", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_verify_scatter": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mds_wipe_sessions": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mempool_debug": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "memstore_device_bytes": "1073741824", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "memstore_page_set": "false", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "memstore_page_size": "65536", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_client_bytes": "134217728", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_client_messages": "512", 2024-09-14T04:03:44.549 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_disabled_modules": "", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_enable_op_tracker": "true", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_max_pg_creating": "1024", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_max_pg_num_change": "128", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_mds_bytes": "134217728", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_mds_messages": "128", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_mon_bytes": "134217728", 2024-09-14T04:03:44.550 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_mon_messages": "128", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_history_duration": "600", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_history_size": "20", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_op_log_threshold": "5", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_osd_bytes": "536870912", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_osd_messages": "8192", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_pool": "true", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_standby_modules": "true", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_stats_period": "5", 2024-09-14T04:03:44.551 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_stats_threshold": "5", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_tick_period": "2", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_allow_pool_delete": "true", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_allow_pool_size_one": "false", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_auth_validate_all_caps": "true", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_bytes": "104857600", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_directed_command_retry": "2", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-14T04:03:44.552 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_hunt_parallel": "3", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_log_interval": "1.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_ping_interval": "10.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_client_target_rank": "-1", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_level": "debug", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_file": "true", 2024-09-14T04:03:44.553 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_journald": "true", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_compact_on_bootstrap": "false", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_compact_on_start": "false", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_compact_on_trim": "true", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_cpu_threads": "4", 2024-09-14T04:03:44.554 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_crush_min_required_version": "hammer", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_daemon_bytes": "419430400", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_data_avail_crit": "5", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_data_avail_warn": "5", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_data_size_warn": "16106127360", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_dump_json": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_dump_transactions": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_extra_checks": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-14T04:03:44.555 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_no_require_reef": "false", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_no_require_squid": "false", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_down_mkfs_grace": "60", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_down_uptime_grace": "60", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_election_default_strategy": "3", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_election_timeout": "5.000000", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_elector_ping_divisor": "2", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_enable_op_tracker": "true", 2024-09-14T04:03:44.556 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_fake_pool_delete": "false", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_force_quorum_join": "false", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_globalid_prealloc": "10000", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_detail_to_clog": "true", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_log_update_period": "5", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_max_detail": "50", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_to_clog": "true", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_to_clog_interval": "600", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_host": "[v2:172.21.15.1:3300/0,v1:172.21.15.1:6789/0] [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.1:3301/0,v1:172.21.15.1:6790/0]", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_host_override": "", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_initial_members": "", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-14T04:03:44.557 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_lease": "5.000000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_log_full_interval": "50", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_log_max": "10000", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_log_max_summary": "50", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_log_epochs": "500", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-14T04:03:44.558 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_osd": "10000", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_pg_per_osd": "10000", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_pool_pg_num": "65536", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mds_force_trim_to": "0", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mds_skip_sanity": "true", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_memory_autotune": "true", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_memory_target": "2147483648", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_beacon_grace": "30", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_digest_period": "5", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_inactive_grace": "60", 2024-09-14T04:03:44.559 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_min_osdmap_epochs": "500", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_complaint_time": "30", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_history_duration": "600", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_history_size": "20", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_history_slow_op_size": "20", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_op_log_threshold": "5", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-14T04:03:44.560 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_auto_mark_in": "false", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_cache_size": "500", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_down_out_interval": "600", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_force_trim_to": "0", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-14T04:03:44.561 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_min_down_reporters": "2", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_report_timeout": "900", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-14T04:03:44.562 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_stuck_threshold": "60", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-14T04:03:44.563 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_probe_timeout": "2.000000", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_reweight_max_change": "0.050000", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_reweight_max_osds": "4", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_scrub_interval": "86400", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_scrub_max_keys": "100", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_scrub_timeout": "300", 2024-09-14T04:03:44.564 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_session_timeout": "300", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_smart_report_timeout": "5", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_stat_smooth_intervals": "6", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_stretch_pool_min_size": "2", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_stretch_pool_size": "4", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_debug": "false", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_provider_kill_at": "0", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_requester_kill_at": "0", 2024-09-14T04:03:44.565 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_sync_timeout": "60.000000", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_target_pg_per_osd": "100", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_tick_interval": "5", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_timecheck_interval": "300.000000", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_use_min_delay_socket": "false", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_older_version_delay": "604800", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-14T04:03:44.566 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_misplaced": "false", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_older_version": "true", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "monmap": "", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-14T04:03:44.567 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "motr_tracing_enabled": "false", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_op_threads": "3", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_cm": "false", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_device_name": "", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_dscp": "96", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_local_gid": "", 2024-09-14T04:03:44.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_port_num": "1", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_sl": "3", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_support_srq": "true", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_rdma_type": "ib", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_async_reap_threshold": "5", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_before_connect": "false", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_ipv4": "true", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_ipv6": "false", 2024-09-14T04:03:44.569 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_msgr1": "true", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_msgr2": "true", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_port_max": "7568", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_port_min": "6800", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_retry_count": "3", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_bind_retry_delay": "5", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_blackhole_client": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_blackhole_mds": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_blackhole_mgr": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_blackhole_mon": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_blackhole_osd": "false", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_client_mode": "crc secure", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_cluster_mode": "crc secure", 2024-09-14T04:03:44.570 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_cluster_type": "", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_compress_secure": "false", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_connection_idle_timeout": "900", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_connection_ready_timeout": "10", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_crc_data": "true", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_crc_header": "true", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_die_on_bad_msg": "false", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_die_on_bug": "true", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_die_on_old_message": "true", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_die_on_skipped_message": "false", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_coremask": "0xF", 2024-09-14T04:03:44.571 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_enable_tso": "true", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_hugepages": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_lro": "true", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_memory_channel": "4", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_pmd": "", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_port_id": "0", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-14T04:03:44.572 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_dump_on_send": "false", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_initial_backoff": "0.200000", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_delay_max": "1.000000", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_delay_type": "", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_network_congestion": "0", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_inject_socket_failures": "0", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_learn_addr_from_peer": "true", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_max_accept_failures": "4", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_max_backoff": "15.000000", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_mon_client_mode": "secure crc", 2024-09-14T04:03:44.573 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_mon_service_mode": "secure crc", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_osd_compress_min_size": "1024", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_osd_compress_mode": "none", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_pq_min_cost": "65536", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_public_type": "", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_service_mode": "crc secure", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_tcp_listen_backlog": "512", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_tcp_nodelay": "true", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_tcp_rcvbuf": "0", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "ms_type": "async+posix", 2024-09-14T04:03:44.574 INFO:teuthology.orchestra.run.smithi001.stdout: "no_config_file": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_completion_locks_per_session": "32", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_inflight_ops": "1024", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_tick_interval": "5.000000", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objecter_timeout": "10.000000", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objectstore_blackhole": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "openssl_engine_opts": "", 2024-09-14T04:03:44.575 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_delay_time": "5.000000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_hist_halflife": "1000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_max_low_ops": "2", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_max_ops": "4", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_agent_slop": "0.020000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_asio_thread_count": "2", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_async_recovery_min_cost": "100", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backfill_scan_max": "512", 2024-09-14T04:03:44.576 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backfill_scan_min": "64", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backoff_on_degraded": "false", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backoff_on_peering": "false", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_backoff_on_unfound": "true", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_beacon_report_interval": "300", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_bench_duration": "30", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_bench_max_block_size": "67108864", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_blkin_trace_all": "false", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-14T04:03:44.577 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_check_for_log_corruption": "false", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_class_update_on_start": "true", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_client_message_cap": "256", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_client_message_size_cap": "524288000", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_client_op_priority": "63", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_client_watch_timeout": "30", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_command_max_records": "256", 2024-09-14T04:03:44.578 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_compact_on_start": "false", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_crush_update_on_start": "true", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_crush_update_weight_set": "true", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_feed_pullee": "-1", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-14T04:03:44.579 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_misdirected_ops": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_no_acting_change": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_no_purge_strays": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_op_order": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_shutdown": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_trim_objects": "false", 2024-09-14T04:03:44.580 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_verify_snaps": "false", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_keys": "1024", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_stride": "524288", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_default_notify_timeout": "30", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_delete_sleep": "0.000000", 2024-09-14T04:03:44.581 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_discard_disconnected_ops": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_ec_partial_reads": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_enable_op_tracker": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_fast_info": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_fast_shutdown": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-14T04:03:44.582 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_function_tracing": "false", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_grace": "20", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_interval": "6", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_min_peers": "10", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_min_size": "2000", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_stale": "600", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_hit_set_max_size": "100000", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_hit_set_min_size": "1000", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-14T04:03:44.583 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_journal_size": "5120", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_kill_backfill_at": "0", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_map_cache_size": "50", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_map_dedup": "true", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_map_message_max": "40", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_map_share_max_epochs": "40", 2024-09-14T04:03:44.584 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_attr_name_len": "100", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_attr_size": "0", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_backfills": "1", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_markdown_count": "5", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_markdown_period": "600", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_object_name_len": "2048", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_object_namespace_len": "256", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_object_size": "134217728", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_pg_blocked_by": "16", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_pg_log_entries": "10000", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_pgls": "1024", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_push_cost": "8388608", 2024-09-14T04:03:44.585 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_push_objects": "10", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_scrubs": "3", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_trimming_pgs": "2", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_write_op_reply_len": "64", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_max_write_size": "90", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-14T04:03:44.586 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_profile": "balanced", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_base": "805306368", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_cache_min": "134217728", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-14T04:03:44.587 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_target": "4294967296", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_target_autotune": "false", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_min_pg_log_entries": "250", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_min_recovery_priority": "0", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mon_report_interval": "5", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_num_cache_shards": "32", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_num_op_tracker_shard": "32", 2024-09-14T04:03:44.588 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_numa_auto_affinity": "true", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_numa_node": "-1", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_numa_prefer_iface": "true", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_objecter_finishers": "1", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_objectstore": "bluestore", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_objectstore_fuse": "false", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_objectstore_tracing": "false", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_complaint_time": "30.000000", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_history_duration": "600", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_history_size": "20", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_history_slow_op_size": "20", 2024-09-14T04:03:44.589 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_log_threshold": "5", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_shards": "0", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_shards_hdd": "5", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_shards_ssd": "8", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_pq_min_cost": "65536", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_queue_cut_off": "high", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_op_thread_timeout": "15", 2024-09-14T04:03:44.590 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_open_classes_on_start": "true", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_os_flags": "0", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_peering_op_priority": "255", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_delete_cost": "1048576", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_delete_priority": "5", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_log_trim_max": "10000", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_log_trim_min": "100", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-14T04:03:44.592 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_crimson": "false", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-14T04:03:44.593 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_flags": "0", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_min_size": "0", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_pg_num": "32", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_pgp_num": "0", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_read_ratio": "70", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_size": "3", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_default_type": "replicated", 2024-09-14T04:03:44.594 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_push_per_object_cost": "1000", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recover_clone_overlap": "true", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_cost": "20971520", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_active": "0", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_max_single_start": "1", 2024-09-14T04:03:44.595 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_op_priority": "3", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_priority": "5", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_sleep": "0.000000", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_repair_during_recovery": "false", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_requested_scrub_priority": "120", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_auto_repair": "false", 2024-09-14T04:03:44.596 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_begin_hour": "0", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_begin_week_day": "0", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_chunk_max": "25", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_chunk_min": "5", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_cost": "52428800", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_during_recovery": "false", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_end_hour": "0", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_end_week_day": "0", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_event_cost": "4096", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-14T04:03:44.597 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_invalid_stats": "true", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_max_preemptions": "5", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_priority": "5", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_retry_after_noscrub": "60", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_retry_delay": "30", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_retry_new_interval": "10", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_retry_pg_state": "60", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_retry_trimming": "10", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_scrub_sleep": "0.000000", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-14T04:03:44.598 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_skip_data_digest": "false", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_smart_report_timeout": "5", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_cost": "1048576", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_priority": "5", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-14T04:03:44.599 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_target_transaction_size": "30", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_tracing": "false", 2024-09-14T04:03:44.600 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_use_stale_snap": "false", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "osdc_blkin_trace_all": "false", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_kill_at": "0", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_max_join_drift": "10", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_min": "500", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_min_wait": "0.050000", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_propose_interval": "1.000000", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_service_trim_max": "500", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_service_trim_min": "250", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_stash_full_interval": "25", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_trim_max": "500", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "paxos_trim_min": "250", 2024-09-14T04:03:44.601 INFO:teuthology.orchestra.run.smithi001.stdout: "perf": "true", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "pid_file": "", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "public_addr": "-", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "public_addrv": "", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "public_bind_addr": "-", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "public_network": "172.21.0.0/20", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "public_network_interface": "", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "qat_compressor_busy_polling": "false", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "qat_compressor_enabled": "false", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "qat_compressor_session_max_number": "256", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "rados_mon_op_timeout": "0", 2024-09-14T04:03:44.602 INFO:teuthology.orchestra.run.smithi001.stdout: "rados_osd_op_timeout": "0", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rados_tracing": "false", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_atime_update_interval": "60", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_balance_parent_reads": "false", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_balance_snap_reads": "false", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_blkin_trace_all": "false", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache": "true", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-14T04:03:44.603 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_policy": "writearound", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_size": "33554432", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_clone_copy_on_read": "false", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_compression_hint": "none", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_concurrent_management_ops": "10", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_clone_format": "auto", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_data_pool": "", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_features": "61", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_format": "2", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_map_options": "", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_order": "22", 2024-09-14T04:03:44.604 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_pool": "rbd", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_stripe_count": "0", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_default_stripe_unit": "0", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_enable_alloc_hint": "true", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_io_scheduler": "simple", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-14T04:03:44.605 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_order": "24", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_pool": "", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_journal_splay_width": "4", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_localize_parent_reads": "false", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_localize_snap_reads": "false", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-14T04:03:44.606 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-14T04:03:44.607 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-14T04:03:44.608 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_mtime_update_interval": "60", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_non_blocking_aio": "true", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_op_thread_timeout": "60", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_op_threads": "1", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_parent_cache_enabled": "false", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_plugins": "", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_bps_burst": "0", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_bps_limit": "0", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_exclude_ops": "0", 2024-09-14T04:03:44.609 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_iops_burst": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_iops_limit": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-14T04:03:44.610 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_read_from_replica_policy": "default", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_skip_partial_discard": "true", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_tracing": "false", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_validate_names": "true", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "rbd_validate_pool": "true", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "restapi_base_url": "", 2024-09-14T04:03:44.611 INFO:teuthology.orchestra.run.smithi001.stdout: "restapi_log_level": "", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_acl_grants_max_num": "100", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_admin_entry": "admin", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_asio_assert_yielding": "false", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_backend_store": "rados", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_barbican_url": "", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_beast_enable_async": "true", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_counters_cache": "false", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-14T04:03:44.612 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_cache_enabled": "true", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_cache_expiry_interval": "900", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_cache_lru_size": "25000", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_config_store": "rados", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_content_length_compat": "false", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_copy_obj_progress": "true", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_cors_rules_max_num": "100", 2024-09-14T04:03:44.613 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_cross_domain_policy": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_addr": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_password": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_kmip_username": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_require_ssl": "true", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-14T04:03:44.614 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_addr": "", 2024-09-14T04:03:44.615 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_auth": "token", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_namespace": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_prefix": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_token_file": "", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_curl_buffersize": "524288", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_curl_low_speed_time": "300", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-14T04:03:44.616 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-14T04:03:44.617 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_log_changes_size": "1000", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_log_num_shards": "128", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_log_window": "30", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-14T04:03:44.618 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_disable_s3select": "false", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-14T04:03:44.619 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dns_name": "", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dns_s3website_name": "", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dynamic_resharding": "true", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_gc_threads": "true", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_lc_threads": "true", 2024-09-14T04:03:44.620 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_mdsearch": "true", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_ops_log": "false", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_quota_threads": "true", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_static_website": "false", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enable_usage_log": "false", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_enforce_swift_acls": "true", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_exit_timeout_secs": "120", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_expose_bucket": "false", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_extended_http_attrs": "", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_filter": "none", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_frontends": "beast port=7480", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_deferred": "50", 2024-09-14T04:03:44.621 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_objs": "32", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_gc_processor_period": "3600", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_init_timeout": "300", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-14T04:03:44.622 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_kafka_connection_idle": "300", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_domain": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_password": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_password_path": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_project": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_tenant": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_token": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_token_path": "", 2024-09-14T04:03:44.623 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_admin_user": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_barbican_domain": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_barbican_password": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_barbican_project": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_barbican_user": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_url": "", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_debug_interval": "-1", 2024-09-14T04:03:44.624 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_lock_max_time": "90", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_max_objs": "32", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_max_rules": "1000", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_max_worker": "3", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lc_thread_delay": "0", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_dnattr": "uid", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_searchfilter": "", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ldap_uri": "", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-14T04:03:44.625 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_log_http_headers": "", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_log_object_name_utc": "false", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_attr_name_len": "0", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_attr_size": "0", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_chunk_size": "4194304", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-14T04:03:44.626 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_listing_results": "5000", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_notify_retries": "10", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_put_param_size": "1048576", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_put_size": "5368709120", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_max_slo_entries": "1000", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_md_log_max_shards": "64", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_mp_lock_max_time": "600", 2024-09-14T04:03:44.627 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_max_gc": "300", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-14T04:03:44.628 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_num_async_rados_threads": "32", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_num_control_oids": "8", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_numa_node": "-1", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_objexp_chunk_size": "100", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_objexp_gc_interval": "600", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-14T04:03:44.629 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_op_thread_timeout": "600", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_op_tracing": "false", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_opa_token": "", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_opa_url": "", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_opa_verify_ssl": "true", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ops_log_rados": "false", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_ops_log_socket_path": "", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_period_push_interval": "2.000000", 2024-09-14T04:03:44.630 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_cache_lanes": "3", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_cache_partitions": "3", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_print_continue": "true", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-14T04:03:44.631 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_rados_tracing": "false", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_realm": "", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_realm_id": "", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_region": "", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_request_uri": "", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_batch_size": "64", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-14T04:03:44.632 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_max_aio": "128", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_num_logs": "16", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_progress_judge_interval": "120", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_progress_judge_ratio": "0.500000", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshard_thread_interval": "600", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_reshardlog_threshold": "30000", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_resolve_cname": "false", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_run_sync_thread": "true", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-14T04:03:44.633 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_scheduler_type": "throttler", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_script_uri": "", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_service_provider_name": "", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_client_id": "", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_client_secret": "", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_entry": "sts", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_key": "sts", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-14T04:03:44.634 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_min_session_duration": "900", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sts_token_introspection_url": "", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_account_in_url": "false", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_auth_entry": "auth", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_auth_url": "", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_custom_header": "", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_need_stats": "true", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_tenant_name": "", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_token_expiration": "86400", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_url": "", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_url_prefix": "swift", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-14T04:03:44.635 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_lease_period": "120", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_thread_pool_size": "512", 2024-09-14T04:03:44.636 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_comment": "", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_createby": "", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_encoding": "", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_flag": "false", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_origin": "", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_torrent_tracker": "", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_trust_forwarded_https": "false", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-14T04:03:44.637 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_usage_max_shards": "32", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_usage_max_user_shards": "1", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_use_opa_authz": "false", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_counters_cache": "false", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_max_buckets": "1000", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_policies_max_num": "100", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-14T04:03:44.638 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_user_unique_email": "true", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_verify_ssl": "true", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zone": "", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zone_id": "", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zonegroup": "", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zonegroup_id": "", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgwlc_auto_session_clear": "true", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_block_size": "4096", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-14T04:03:44.639 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_size": "536870912", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_index_type": "binary_search", 2024-09-14T04:03:44.640 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_partition_filters": "false", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_perf": "false", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "run_dir": "/var/run/ceph", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_block_create": "true", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_cache_lru_size": "67108864", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_default_max_object_size": "16777216", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-14T04:03:44.641 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_device_size": "53687091200", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_full_integrity_check": "false", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_journal_batch_capacity": "16", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_main_device_type": "SSD", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-14T04:03:44.642 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "seastore_segment_size": "67108864", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "set_keepcaps": "false", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "setgroup": "ceph", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "setuser": "ceph", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "setuser_match_path": "", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "thp": "false", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "threadpool_default_timeout": "60", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "throttler_perf_counter": "true", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "tmp_dir": "/tmp", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "uadk_compressor_enabled": "false", 2024-09-14T04:03:44.643 INFO:teuthology.orchestra.run.smithi001.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-14T04:03:44.644 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:03:45.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-14T04:03:45.485 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:45 smithi012 ceph-mon[101180]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:45.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:45 smithi001 ceph-mon[107399]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:45.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:45 smithi001 ceph-mon[112165]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:46.945 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:46] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.51.0" 2024-09-14T04:03:47.179 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a: {} 2024-09-14T04:03:47.198 INFO:teuthology.orchestra.run.smithi001.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:03:47.199 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b: {} 2024-09-14T04:03:47.199 INFO:teuthology.orchestra.run.smithi001.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:03:47.199 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c: {} 2024-09-14T04:03:47.199 INFO:teuthology.orchestra.run.smithi001.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:03:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2427036941' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:03:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2427036941' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:03:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2427036941' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:03:47.943 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-14T04:03:47.943 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-14T04:03:47.943 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:03:47.944 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:03:47.944 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-14T04:03:47.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-14T04:03:47.944 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-14T04:03:47.945 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:03:47.945 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:03:47.945 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-14T04:03:47.945 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:03:47.945 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:03:47.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-14T04:03:47.946 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-14T04:03:47.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:47.946 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-14T04:03:47.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:03:47.947 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-14T04:03:47.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:03:47.947 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-14T04:03:47.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:03:47.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:47.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:48.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:48.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:48.203 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:48.211 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:48.215 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:48.228 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:48.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:48.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:48.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:48.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:48.478 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:48 smithi001 ceph-mon[107399]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:48.479 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:48 smithi001 ceph-mon[112165]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:48.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:48 smithi012 ceph-mon[101180]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:48.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:48.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:48.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:48.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:48.859 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:48.881 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:48.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:48.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:49.006 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:49.108 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:49.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:49.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:49.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:49.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:49.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:49.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:49.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:49 smithi012 ceph-mon[101180]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:49.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:49 smithi001 ceph-mon[112165]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:49.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:49 smithi001 ceph-mon[107399]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:49.609 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:49.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:49.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:49.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:49.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:49.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:49.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:50.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:50.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:50.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:50.240 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:50.263 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:50.365 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:50.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:50.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:50.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:50.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:50.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:50.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:50.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:50.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:50.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:50.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:50.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:50.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:50.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:51.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:51.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:51.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:51.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:51.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:51.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:51.376 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:51.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:51 smithi001 ceph-mon[107399]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:51 smithi001 ceph-mon[112165]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:51.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:51.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:51.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:51.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:51.758 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:51.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:51 smithi012 ceph-mon[101180]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:51.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:51.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:51.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:52.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:52.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:52.128 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:03:52.128 INFO:teuthology.orchestra.run.smithi001.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-14T04:03:52.128 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:03:52.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:52.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:52.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:52.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:52.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:52.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:52.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:52.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:52.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:52.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:52.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:52.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:52.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:52.778 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:52.779 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:52.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:52.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:52.919 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:52.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:52.992 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:03:52.992 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:03:52.992 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:03:52.992 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:03:53.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:53.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:53.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:53.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:53.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:53.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:53.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:53.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:53.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:53.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:53.591 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:53 smithi001 ceph-mon[107399]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:53.591 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:53 smithi001 ceph-mon[112165]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:53.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:53.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:53.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:53.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:53.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:53 smithi012 ceph-mon[101180]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:53.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:53.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:53.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:54.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:54.053 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:54.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:03:54.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:54.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:54.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:54.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:54.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:54.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:54.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:54.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:54.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:54.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:54.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:54.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:54.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:54.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:54.944 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:54.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:54.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:55.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.070 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:55.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:55.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:55.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:55.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:55.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:55.425 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:55.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:55 smithi001 ceph-mon[112165]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:55.436 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:55 smithi001 ceph-mon[107399]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:55.449 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:55.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:55.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:55.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:55.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:55.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:55.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:55 smithi012 ceph-mon[101180]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:55.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:55.956 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:55.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:56.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:56.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:56.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:56.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:56.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:56.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:56.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:56.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:56.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:56.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:56.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:56.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:56.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:56.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:56.839 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:56.932 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:03:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:03:56] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.51.0" 2024-09-14T04:03:56.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:56.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:56.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:57.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:57.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:57.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:57.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:57.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:57.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:57.352 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:57.391 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:03:57.425 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:03:57.425 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:03:57.442 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:03:57.443 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:03:57.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:57.459 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:03:57.459 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:03:57.476 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:03:57.476 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:03:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:57.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:57.493 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:03:57.493 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:03:57.511 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:03:57.511 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:03:57.527 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:03:57.527 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:03:57.544 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:03:57.545 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:03:57.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:57.605 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:57.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:57.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:57 smithi001 ceph-mon[107399]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:57.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:03:57.713 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:57 smithi001 ceph-mon[112165]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:57.713 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:03:57.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:57.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:57 smithi012 ceph-mon[101180]: pgmap v444: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:03:57.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:57.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:03:57.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:57.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:03:58.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:58.104 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:58.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:58.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:58.233 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:58.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:58.359 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:58.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:58.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:58.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:58.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:58.684 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[112165]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1652036850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[107399]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T04:03:58.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1652036850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:03:58.713 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:58.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:58.742 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:58.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:58 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:03:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2765760963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:03:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:58 smithi012 ceph-mon[101180]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T04:03:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1652036850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:03:58.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:58.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:03:58.991 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:58.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:03:59.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:59.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:59.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:03:59.244 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:59.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:03:59.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:03:59.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:03:59.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:03:59.498 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:03:59.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:03:59.626 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:03:59.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:03:59.751 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:03:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:03:59.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:03:59.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:03:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:03:59.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:03:59.888 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_4' created 2024-09-14T04:03:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:59 smithi001 ceph-mon[112165]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:03:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:03:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:59 smithi001 ceph-mon[107399]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:03:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:03:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:03:59.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:00.007 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:00.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:04:00.109 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:00.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:00.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:00.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:00.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:59 smithi012 ceph-mon[101180]: pgmap v446: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:03:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:04:00.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:00.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:00.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:00.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:00.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:00.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:00.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:00.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:04:00.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:00.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:00.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:00.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:00.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:00.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:01.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:01.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:01.073 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:04:01.074 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:04:01.074 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-14T04:04:01.074 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:04:01.074 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:04:01.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-14T04:04:01.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:01.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:04:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:00 smithi001 ceph-mon[107399]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T04:04:01.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:04:01.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:00 smithi001 ceph-mon[112165]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T04:04:01.250 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:04:01.250 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:04:01.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:01.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:01.313 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:04:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1380666998' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:04:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:00 smithi012 ceph-mon[101180]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T04:04:01.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:01.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:01.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:01.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:01.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:01.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:01.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:01.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:01.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:01.792 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:01.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:01.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:01.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:01 smithi001 ceph-mon[107399]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T04:04:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:01 smithi001 ceph-mon[107399]: pgmap v449: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:01 smithi001 ceph-mon[112165]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T04:04:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:01 smithi001 ceph-mon[112165]: pgmap v449: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:02.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:04:02.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:02.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:01 smithi012 ceph-mon[101180]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T04:04:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:01 smithi012 ceph-mon[101180]: pgmap v449: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:02.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:02.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:02.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:02.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:02.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:02.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:02.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:02.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-14T04:04:02.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:02.614 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:02.642 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:02.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:02.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:02.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:02.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:02.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:02.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:02.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:03.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:03.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:03.083 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:04:03.083 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:04:03.083 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:04:03.092 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:02 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:03.092 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:02 smithi012 ceph-mon[101180]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T04:04:03.128 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:02 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:03.129 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:02 smithi001 ceph-mon[107399]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T04:04:03.129 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:02 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:03.129 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:02 smithi001 ceph-mon[112165]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T04:04:03.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:03.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:03.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:03.330 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:03.335 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:03.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:03.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:03.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:03.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:03.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:03.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:03.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:03.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:03.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:03.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:03.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:04.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:04.080 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:03 smithi012 ceph-mon[101180]: pgmap v451: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:04.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:04.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:03 smithi001 ceph-mon[107399]: pgmap v451: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:03 smithi001 ceph-mon[112165]: pgmap v451: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:04.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:04.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:04.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:04.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:04.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:04.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:04.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:04.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:04.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:04.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:04.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:04.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:04.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:05.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:05.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:05.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:05.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:05.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:05.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:05.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:05.457 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:05.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:05.558 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:05.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:05.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:05.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:05.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:05.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:05.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:05.936 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:05.960 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:05.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:05.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:05.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[112165]: pgmap v452: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:05 smithi001 ceph-mon[107399]: pgmap v452: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:06.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:06.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:06.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:06.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:06.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:05 smithi012 ceph-mon[101180]: pgmap v452: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:06.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:06.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:06.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:06.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:06.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:06.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:06.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:06.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:06.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:06.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:06.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:06.968 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:06] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-09-14T04:04:06.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:06.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:06.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:07.084 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:07.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:07.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:07.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:07.311 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:04:07.311 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:04:07.312 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:04:07.330 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:04:07.330 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:04:07.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:07.356 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:04:07.357 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:04:07.364 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:07.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:07.373 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:04:07.373 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:04:07.389 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:04:07.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:04:07.406 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:04:07.407 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:04:07.423 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:04:07.423 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:04:07.440 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:04:07.440 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:04:07.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:07.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:07.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1125560344' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:07 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3890076033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:07 smithi012 ceph-mon[101180]: pgmap v453: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:07.612 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:07.714 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:07.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1125560344' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3890076033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:07.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[112165]: pgmap v453: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1125560344' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:07.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3890076033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:07.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:07 smithi001 ceph-mon[107399]: pgmap v453: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:07.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:07.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:07.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:07.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:07.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:07.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:07.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:08.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:08.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:08.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:08.244 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:08.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:08.365 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:08.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:2 8.6 8:62d4c190:::benchmark_data_smithi001_125658_object1:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.717678+0000", 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.65730679900003, 2024-09-14T04:04:08.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27237508599999999, 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 2 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717678+0000", 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717678+0000", 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717680+0000", 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9470000000000002e-06 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717740+0000", 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0411000000000001e-05 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717755+0000", 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5173e-05 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717765+0000", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9890000000000004e-06 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717870+0000", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000104978 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717882+0000", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1735e-05 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.982663+0000", 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26478111199999999 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.982780+0000", 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117241 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.982943+0000", 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016292800000000001 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.985805+0000", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028620270000000001 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.986258+0000", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000452239 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.989960+0000", 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037020310000000002 2024-09-14T04:04:08.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990010+0000", 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0096000000000003e-05 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990053+0000", 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3179000000000001e-05 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:4 8.e 8:731e2b04:::benchmark_data_smithi001_125658_object3:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.717799+0000", 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.65718563299998, 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27749575100000001, 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.387 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 4 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717799+0000", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717799+0000", 2024-09-14T04:04:08.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717800+0000", 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.61e-07 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717882+0000", 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2090999999999997e-05 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717893+0000", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1362000000000001e-05 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717901+0000", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4050000000000003e-06 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718071+0000", 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017047000000000001 2024-09-14T04:04:08.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718079+0000", 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.807e-06 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990212+0000", 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27213353200000001 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990654+0000", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00044182800000000001 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990789+0000", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013452599999999999 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.992432+0000", 2024-09-14T04:04:08.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001642849 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.993863+0000", 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014309730000000001 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995148+0000", 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001285069 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995191+0000", 2024-09-14T04:04:08.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2975000000000001e-05 2024-09-14T04:04:08.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995295+0000", 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000104103 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:8 8.6 8:6508e312:::benchmark_data_smithi001_125658_object7:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.717934+0000", 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.65705052599998, 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27242078199999997, 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.395 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 8 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717934+0000", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717934+0000", 2024-09-14T04:04:08.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717935+0000", 2024-09-14T04:04:08.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.61e-07 2024-09-14T04:04:08.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.717990+0000", 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5464000000000003e-05 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718005+0000", 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5166e-05 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718012+0000", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3230000000000003e-06 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718102+0000", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0111000000000005e-05 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.718107+0000", 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8049999999999997e-06 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.983298+0000", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26519129000000002 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.983404+0000", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106579 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.984645+0000", 2024-09-14T04:04:08.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001240133 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.987186+0000", 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025416950000000001 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.987699+0000", 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000513124 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990263+0000", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025633359999999998 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990316+0000", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3463e-05 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990355+0000", 2024-09-14T04:04:08.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8631999999999999e-05 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:32 8.6 8:6400ce1c:::benchmark_data_smithi001_125658_object31:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.724849+0000", 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.65013547799998, 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27019011799999998, 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.406 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.407 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.407 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.407 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.407 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 32 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724849+0000", 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724849+0000", 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724852+0000", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.146e-06 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724912+0000", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0285999999999998e-05 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724951+0000", 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8452999999999999e-05 2024-09-14T04:04:08.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.724969+0000", 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8386999999999999e-05 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725064+0000", 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4359999999999998e-05 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725133+0000", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8874000000000005e-05 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990388+0000", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26525494500000002 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990499+0000", 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000111055 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990709+0000", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000210373 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.992460+0000", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0017513699999999999 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.993868+0000", 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00140745 2024-09-14T04:04:08.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.994821+0000", 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000953216 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.994995+0000", 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017449800000000001 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995039+0000", 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3705000000000003e-05 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:38 8.6 8:66f6482e:::benchmark_data_smithi001_125658_object37:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.725580+0000", 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.64940466600001, 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27003727599999999, 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.414 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 38 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725580+0000", 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725580+0000", 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725581+0000", 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.345e-06 2024-09-14T04:04:08.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725657+0000", 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5909999999999997e-05 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725674+0000", 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7365999999999999e-05 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725691+0000", 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6078000000000001e-05 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725781+0000", 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0673000000000004e-05 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725789+0000", 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2910000000000002e-06 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.992508+0000", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26671924600000002 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.993397+0000", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00088938500000000002 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.993557+0000", 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015942199999999999 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995057+0000", 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015008090000000001 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995374+0000", 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000316927 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995511+0000", 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013647900000000001 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995562+0000", 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0822999999999998e-05 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995617+0000", 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5522000000000001e-05 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.421 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:39 8.c 8:3c8a943b:::benchmark_data_smithi001_125658_object38:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.725718+0000", 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.64926699300003, 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27030565200000001, 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 39 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.422 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725718+0000", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725718+0000", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725718+0000", 2024-09-14T04:04:08.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2900000000000003e-07 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725781+0000", 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2879999999999994e-05 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725792+0000", 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0601000000000001e-05 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725804+0000", 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2724e-05 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725923+0000", 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118353 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.725930+0000", 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6399999999999997e-06 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.987398+0000", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26146745599999999 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.987541+0000", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014313300000000001 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.987730+0000", 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018847800000000001 2024-09-14T04:04:08.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.989482+0000", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001752036 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990389+0000", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00090790599999999999 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995940+0000", 2024-09-14T04:04:08.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005550925 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.995986+0000", 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5309000000000003e-05 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.996023+0000", 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7481999999999998e-05 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.428 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:43 8.c 8:33bc9a47:::benchmark_data_smithi001_125658_object42:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:07.727419+0000", 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.64756542399999, 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.270806777, 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.429 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 43 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727419+0000", 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727419+0000", 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727420+0000", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1179999999999999e-06 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727480+0000", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9555000000000002e-05 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727499+0000", 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9219999999999999e-05 2024-09-14T04:04:08.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727511+0000", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2261e-05 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727576+0000", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4924000000000004e-05 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.727584+0000", 2024-09-14T04:04:08.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.988060+0000", 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.260475864 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.988188+0000", 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000128553 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:04:08.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.988404+0000", 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021541799999999999 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.989486+0000", 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001081799 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.990888+0000", 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014022939999999999 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.998065+0000", 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0071767549999999999 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.998146+0000", 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1520999999999994e-05 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:07.998226+0000", 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9719e-05 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:58877 8.1 8:8954afa3:::benchmark_data_smithi001_125658_object58876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e105)", 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:33.236886+0000", 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 395.138098856, 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0115610909999999, 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.436 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 58877 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236886+0000", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236886+0000", 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236887+0000", 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.584e-06 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236924+0000", 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6285999999999998e-05 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236936+0000", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.206e-05 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236948+0000", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2644e-05 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237263+0000", 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00031509399999999998 2024-09-14T04:04:08.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237273+0000", 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1770000000000004e-06 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.247549+0000", 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.01027687 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.248447+0000", 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00089737599999999997 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:58879 8.1 8:84f23e76:::benchmark_data_smithi001_125658_object58878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e105)", 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:33.236971+0000", 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 395.13801386300003, 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.011768051, 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:08.441 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 58879 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236971+0000", 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236971+0000", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236971+0000", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9999999999999997e-07 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.236994+0000", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2459e-05 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237003+0000", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9060000000000008e-06 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237013+0000", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0696000000000001e-05 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237300+0000", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000286316 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237306+0000", 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1670000000000004e-06 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.248662+0000", 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0113564420000001 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.248739+0000", 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6464999999999994e-05 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:58883 8.1 8:8c2b73d7:::benchmark_data_smithi001_125658_object58882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e105)", 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:33.237024+0000", 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 395.13796034199999, 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0224727899999999, 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.446 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 58883 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237024+0000", 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237024+0000", 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237025+0000", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.75e-07 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237046+0000", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.124e-05 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237053+0000", 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2699999999999999e-06 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237060+0000", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3239999999999999e-06 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237352+0000", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000291938 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237358+0000", 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3960000000000003e-06 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.248776+0000", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.011418535 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for degraded object", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.249151+0000", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00037445799999999998 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.259409+0000", 2024-09-14T04:04:08.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010257703 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.259497+0000", 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8350999999999995e-05 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:58885 8.1 8:8f922e0a:::benchmark_data_smithi001_125658_object58884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e105)", 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:33.237069+0000", 2024-09-14T04:04:08.451 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 395.137915609, 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.025880911, 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 58885 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237069+0000", 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237069+0000", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237070+0000", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9599999999999999e-07 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237091+0000", 2024-09-14T04:04:08.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1596000000000001e-05 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237100+0000", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.918e-06 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237106+0000", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7420000000000003e-06 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237382+0000", 2024-09-14T04:04:08.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027632900000000002 2024-09-14T04:04:08.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237388+0000", 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0179999999999998e-06 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.249226+0000", 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.011837385 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.249643+0000", 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00041739499999999998 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2", 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.250171+0000", 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00052842500000000001 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.259541+0000", 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0093690849999999992 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.262706+0000", 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00316592 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.262843+0000", 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013681399999999999 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.262950+0000", 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106588 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.458 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:58898 8.1 8:86bfca1e:::benchmark_data_smithi001_125658_object58897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e105)", 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:33.237115+0000", 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 395.13786927699999, 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0263120429999999, 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 58898 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237115+0000", 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237115+0000", 2024-09-14T04:04:08.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237116+0000", 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0200000000000002e-07 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237168+0000", 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2139999999999999e-05 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237176+0000", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2020000000000004e-06 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237182+0000", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9959999999999999e-06 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237439+0000", 2024-09-14T04:04:08.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000256817 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:33.237445+0000", 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3770000000000001e-06 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.253855+0000", 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0164100760000001 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.254054+0000", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019911899999999999 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.254223+0000", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000168849 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.261324+0000", 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007100294 2024-09-14T04:04:08.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.263264+0000", 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001940841 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.263326+0000", 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1422999999999998e-05 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:34.263427+0000", 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101407 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89073 8.1 8:83ec98f2:::benchmark_data_smithi001_125658_object89072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:55.634818+0000", 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 372.74016683100001, 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.758949852, 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.466 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89073 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634818+0000", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634818+0000", 2024-09-14T04:04:08.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634819+0000", 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7570000000000001e-06 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634875+0000", 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5050999999999999e-05 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634888+0000", 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3567000000000001e-05 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634899+0000", 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0869999999999999e-05 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.640623+0000", 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057243839999999999 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.640638+0000", 2024-09-14T04:04:08.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4837e-05 2024-09-14T04:04:08.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:08.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:08.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[107399]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T04:04:08.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:08 smithi001 ceph-mon[112165]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T04:04:08.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.377059+0000", 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.736420353 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.377209+0000", 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015067600000000001 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.377323+0000", 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114162 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.384945+0000", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0076217610000000003 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.389533+0000", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045883290000000004 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.392585+0000", 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0030514029999999998 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.392649+0000", 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3928000000000002e-05 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.393768+0000", 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0011187739999999999 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89076 8.1 8:8161c32b:::benchmark_data_smithi001_125658_object89075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-14T04:04:08.474 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:55.634918+0000", 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 372.740066743, 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7627884649999999, 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89076 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.475 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634918+0000", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634918+0000", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634919+0000", 2024-09-14T04:04:08.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1699999999999997e-07 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634953+0000", 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4728999999999999e-05 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634964+0000", 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0523000000000001e-05 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634971+0000", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2960000000000001e-06 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.640767+0000", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057960319999999996 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.640776+0000", 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8210000000000008e-06 2024-09-14T04:04:08.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.381443+0000", 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7406673590000001 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.381583+0000", 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013936999999999999 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:04:08.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.381759+0000", 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000176754 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.385131+0000", 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033712999999999998 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.394228+0000", 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0090972320000000002 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.396448+0000", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022205380000000002 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.396509+0000", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0545999999999999e-05 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:57.397706+0000", 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0011972479999999999 2024-09-14T04:04:08.481 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194966 8.e 8:7ed8c295:::benchmark_data_smithi001_125658_object69727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.258334+0000", 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.116650265, 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.46065803100000002, 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194966 2024-09-14T04:04:08.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258334+0000", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258334+0000", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258335+0000", 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.035e-06 2024-09-14T04:04:08.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258340+0000", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3850000000000004e-06 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258351+0000", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1258e-05 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258358+0000", 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8120000000000002e-06 2024-09-14T04:04:08.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258415+0000", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6827999999999999e-05 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258519+0000", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000104818 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.258626+0000", 2024-09-14T04:04:08.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106196 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.262857+0000", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042312280000000001 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.264110+0000", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001253578 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.718901+0000", 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.454790309 2024-09-14T04:04:08.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.718940+0000", 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9708e-05 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.718992+0000", 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1876000000000003e-05 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.487 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194975 8.e 8:757dee2f:::benchmark_data_smithi001_125658_object69736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.274134+0000", 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.10085021, 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.46138622600000001, 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.488 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194975 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274134+0000", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274134+0000", 2024-09-14T04:04:08.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274137+0000", 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6350000000000002e-06 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274146+0000", 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7120000000000006e-06 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274169+0000", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3521999999999999e-05 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274179+0000", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0066e-05 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274237+0000", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7831999999999998e-05 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274328+0000", 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0674999999999995e-05 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.274404+0000", 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5947999999999999e-05 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.278258+0000", 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038545329999999998 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.278607+0000", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034840499999999998 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735435+0000", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.45682864400000001 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735481+0000", 2024-09-14T04:04:08.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5321000000000002e-05 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735521+0000", 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9932999999999999e-05 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194981 8.e 8:74640a77:::benchmark_data_smithi001_125658_object69742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.280042+0000", 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.09494262699999, 2024-09-14T04:04:08.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.45557856299999999, 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194981 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280042+0000", 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280042+0000", 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280044+0000", 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7430000000000001e-06 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280051+0000", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1879999999999996e-06 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280071+0000", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9930000000000001e-05 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280080+0000", 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9919999999999996e-06 2024-09-14T04:04:08.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280139+0000", 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9633999999999999e-05 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280302+0000", 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000162602 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.280444+0000", 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014208499999999999 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.283805+0000", 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033610089999999999 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.284662+0000", 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00085672999999999999 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735553+0000", 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.45089158600000001 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735590+0000", 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6874e-05 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735620+0000", 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0190000000000001e-05 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194983 8.e 8:7c518b44:::benchmark_data_smithi001_125658_object69744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.285291+0000", 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.08969393000001, 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.45043799800000001, 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.501 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194983 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285291+0000", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285291+0000", 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285292+0000", 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.561e-06 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285298+0000", 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.874e-06 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285313+0000", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5316999999999999e-05 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285322+0000", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2230000000000007e-06 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285952+0000", 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00063056799999999997 2024-09-14T04:04:08.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.286041+0000", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9251000000000003e-05 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.286129+0000", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7940000000000002e-05 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.292559+0000", 2024-09-14T04:04:08.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064292810000000002 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.292625+0000", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5969000000000005e-05 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735650+0000", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.44302575799999999 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735699+0000", 2024-09-14T04:04:08.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8327e-05 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735729+0000", 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9929000000000001e-05 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194985 8.e 8:738fe499:::benchmark_data_smithi001_125658_object69746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.285377+0000", 2024-09-14T04:04:08.507 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.08960798000001, 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.45046504199999998, 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194985 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285377+0000", 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285377+0000", 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285377+0000", 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3099999999999998e-07 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285379+0000", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.385e-06 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285386+0000", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8889999999999996e-06 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.285392+0000", 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5829999999999997e-06 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.289272+0000", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038806119999999999 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.289408+0000", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135115 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.289536+0000", 2024-09-14T04:04:08.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012873999999999999 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.292671+0000", 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031349889999999999 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.292951+0000", 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027966500000000001 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735760+0000", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.44280942000000001 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735803+0000", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2724e-05 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735842+0000", 2024-09-14T04:04:08.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8488999999999997e-05 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:194989 8.e 8:7f61d65a:::benchmark_data_smithi001_125658_object69750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:00:06.297850+0000", 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 242.07713482599999, 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.44112992299999998, 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:08.514 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 194989 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297850+0000", 2024-09-14T04:04:08.515 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297850+0000", 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297852+0000", 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9759999999999998e-06 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297859+0000", 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0890000000000002e-06 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297878+0000", 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9420999999999999e-05 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:08.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297889+0000", 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0548e-05 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.297963+0000", 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4303999999999999e-05 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.298053+0000", 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9454000000000001e-05 2024-09-14T04:04:08.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.298136+0000", 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3654999999999998e-05 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.301239+0000", 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031030179999999999 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.301618+0000", 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00037843700000000002 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.735879+0000", 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.43426102700000002 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.738919+0000", 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0030405639999999999 2024-09-14T04:04:08.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:00:06.738980+0000", 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0430000000000002e-05 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:08.521 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:04:08.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:08.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:08 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:08 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:08 smithi012 ceph-mon[101180]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T04:04:08.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:08.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:08.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:08.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:08.833 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:04:08.833 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:04:08.834 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-14T04:04:08.834 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-14T04:04:08.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:08.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:08.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:09.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:09.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:09.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:09.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:09.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:09.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:09.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:09.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:09.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:09.555 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:09 smithi012 ceph-mon[101180]: pgmap v455: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:09.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:09.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:09.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:09 smithi001 ceph-mon[107399]: pgmap v455: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:09.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:09 smithi001 ceph-mon[112165]: pgmap v455: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:09.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:09.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:09.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:09.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:09.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:10.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:10.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:10.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:10.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:04:10.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:10.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:10.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:10.317 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:10.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:10.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:10.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:10.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:10.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:10.676 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:10.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:10.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:10.826 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:10.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:10.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:10.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:10.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:11.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:11.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:11.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:04:11.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:11.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:11.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:11.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:11.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:11.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:11.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:11.589 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:11.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:11.627 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:11.698 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:11.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:11 smithi001 ceph-mon[107399]: pgmap v456: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:11.714 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:11 smithi001 ceph-mon[112165]: pgmap v456: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:11.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:11.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:11.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:11.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:11 smithi012 ceph-mon[101180]: pgmap v456: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:11.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:11.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:11.976 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:11.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:12.078 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:12.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:12.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:12.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:12.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:12.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:12.355 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-14T04:04:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.356 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:12.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:12.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:12.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:04:12.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:12.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:12.613 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:12.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:12.736 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:12.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:12.843 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:12.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:12.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:12.994 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:12.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:12.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:13.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:13.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:13.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:13.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:13.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:13.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[112165]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[112165]: pgmap v458: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[107399]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T04:04:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:13 smithi001 ceph-mon[107399]: pgmap v458: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:13.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:13.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:13.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:13.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:13.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:13.754 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:13 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4274193561' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:13 smithi012 ceph-mon[101180]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T04:04:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:13 smithi012 ceph-mon[101180]: pgmap v458: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:13.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:13.880 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:13.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:13.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:14.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:14.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:14.131 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:14.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:14.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:14.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:14.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:14.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:14.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:14.487 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:14.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:14.545 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:14 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:14.546 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:14 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:14.581 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:14.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:14.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:14.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:14.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:14 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:14.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:14.890 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:14.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:14.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:15.019 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:15.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:15.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:15.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:15.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:15.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:15.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:15.397 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:15 smithi001 ceph-mon[112165]: pgmap v459: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:15 smithi001 ceph-mon[107399]: pgmap v459: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:15.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:15.522 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:15.569 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-14T04:04:15.572 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:04:15.572 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:04:15.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:15.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:15.747 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:15.772 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:15.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:04:15.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:04:15.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_156540 2024-09-14T04:04:15.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:04:15.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:04:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:15 smithi012 ceph-mon[101180]: pgmap v459: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:04:15.878 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:15.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:15.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:16.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:16.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:16.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:16.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:16.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:16.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:16.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:16.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:16.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:16.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:16.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:16.637 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:04:16.637 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:04:16.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:3253 9.e e139/135 9:753f5c29:::benchmark_data_smithi001_156540_object3252:head v 139'216, mlcod=139'214)", 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:16.621480+0000", 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.00061086799999999998, 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00063807200000000005, 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621480+0000", 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:16.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621480+0000", 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621481+0000", 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2799999999999995e-07 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621505+0000", 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4491e-05 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621516+0000", 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.082e-05 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:16.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621523+0000", 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8700000000000003e-06 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621605+0000", 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2188000000000007e-05 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621623+0000", 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8355e-05 2024-09-14T04:04:16.642 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:3255 9.3 e139/135 9:c7910584:::benchmark_data_smithi001_156540_object3254:head v 139'198, mlcod=139'196)", 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:16.621611+0000", 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.00047964500000000001, 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00056616999999999995, 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:04:16.643 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621611+0000", 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621611+0000", 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:16.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621612+0000", 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.75e-07 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621640+0000", 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8004000000000002e-05 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621648+0000", 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1009999999999992e-06 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621654+0000", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6340000000000003e-06 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621961+0000", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030658299999999997 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621986+0000", 2024-09-14T04:04:16.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5012999999999999e-05 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:3254 9.0 9:042cce31:::benchmark_data_smithi001_156540_object3253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:16.620883+0000", 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0012075160000000001, 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0013595669999999999, 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:04:16.647 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 3254 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.620883+0000", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.620883+0000", 2024-09-14T04:04:16.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.620884+0000", 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.252e-06 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.620990+0000", 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105931 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621007+0000", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6339000000000001e-05 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621015+0000", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2179999999999999e-06 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621077+0000", 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2058999999999997e-05 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621192+0000", 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114842 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:16.621320+0000", 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012799600000000001 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:16.651 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:04:16.652 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 3 2024-09-14T04:04:16.652 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:04:16.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:16.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:16.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3852 3836 239.813 239.75 0.00449419 0.00415692 2024-09-14T04:04:16.855 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:16.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:16.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:16] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-09-14T04:04:17.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:17.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:17.140 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:17.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:17.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:17.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:17.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:17.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:17.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:17.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:17.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:17.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:17 smithi001 ceph-mon[107399]: pgmap v460: 89 pgs: 89 active+clean; 57 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-14T04:04:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:17 smithi001 ceph-mon[112165]: pgmap v460: 89 pgs: 89 active+clean; 57 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-14T04:04:17.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:17.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:17.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:17.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 7513 7497 234.302 228.812 0.00504513 0.00426116 2024-09-14T04:04:17.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:17.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:17 smithi012 ceph-mon[101180]: pgmap v460: 89 pgs: 89 active+clean; 57 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-14T04:04:17.887 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:18.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-14T04:04:18.187 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:18.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:18.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:18.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:18.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:18.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:18.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:18.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:18.658 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:18.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:18.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 10848 10832 225.673 208.438 0.00478672 0.00442311 2024-09-14T04:04:18.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:18.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:18.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:18.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:18.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:19.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:19.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:19.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:19.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:19.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:19.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:19.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:19.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:19.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:19.711 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:19.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:19 smithi001 ceph-mon[107399]: pgmap v461: 89 pgs: 89 active+clean; 149 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2024-09-14T04:04:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:19 smithi001 ceph-mon[112165]: pgmap v461: 89 pgs: 89 active+clean; 149 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2024-09-14T04:04:19.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:19.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 15 13616 13601 212.515 173.062 0.00396547 0.00469968 2024-09-14T04:04:19.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:19 smithi012 ceph-mon[101180]: pgmap v461: 89 pgs: 89 active+clean; 149 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2024-09-14T04:04:19.875 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:19.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:19.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:20.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:20.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:20.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:20.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:20.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:20.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:20.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:20.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:20.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:04:20.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:20.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:20.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:20.681 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:04:20.682 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:04:20.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:20.718 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:04:20.718 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:04:20.724 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:20.753 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:04:20.753 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:04:20.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 16421 16405 205.058 175.25 0.0056381 0.00486774 2024-09-14T04:04:20.794 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:04:20.795 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:04:20.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:20.827 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:04:20.827 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:04:20.861 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:04:20.861 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:04:20.862 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:20.897 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:04:20.897 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:04:20.935 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:20.935 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:20.936 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2053740019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:20.936 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:04:20.936 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:04:20.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:20.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:20.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2053740019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:20.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:20.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:20.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:21.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:21.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:21.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:21.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:21.283 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:21.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2053740019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:21.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:21.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:21.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.552 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:21.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:21.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:21.686 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:21.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 18608 18592 193.66 136.688 0.00651787 0.00515794 2024-09-14T04:04:21.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:21.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:21.927 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:04:21.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:21.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:21.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:22.013 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3900788995' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.013 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:21 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.018 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3900788995' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.018 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:21 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:22.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:22.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:22.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:22.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3900788995' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:21 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:04:22.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:22.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:22.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:22.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:22.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:22.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:22.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 21167 21151 188.84 159.938 0.00520486 0.00528933 2024-09-14T04:04:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:22.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:22.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:22.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:22.998 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:23.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:23.050 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:23.077 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:23.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:23.189 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:23.200 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[107399]: pgmap v462: 89 pgs: 89 active+clean; 830 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:04:23.200 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:23.200 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:04:23.201 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[107399]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T04:04:23.201 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[112165]: pgmap v462: 89 pgs: 89 active+clean; 830 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:04:23.201 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:23.201 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:04:23.201 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:22 smithi001 ceph-mon[112165]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T04:04:23.291 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:23.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:22 smithi012 ceph-mon[101180]: pgmap v462: 89 pgs: 89 active+clean; 830 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:04:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:22 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:22 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:04:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:22 smithi012 ceph-mon[101180]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T04:04:23.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:23.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:23.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:23.600 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:23.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:23.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:23.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 23750 23734 185.413 161.438 0.0044765 0.00538754 2024-09-14T04:04:23.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:23.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:23.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:23.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:24.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:24.052 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:04:24.120 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:24.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:24.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:23 smithi001 ceph-mon[112165]: pgmap v464: 89 pgs: 89 active+clean; 1008 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-14T04:04:24.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:23 smithi001 ceph-mon[107399]: pgmap v464: 89 pgs: 89 active+clean; 1008 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-14T04:04:24.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:24.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:24.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:23 smithi012 ceph-mon[101180]: pgmap v464: 89 pgs: 89 active+clean; 1008 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-14T04:04:24.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:24.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:24.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:24.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:24.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:24.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:24.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 26271 26255 182.292 157.562 0.00717087 0.00547936 2024-09-14T04:04:24.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:24.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:24.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:04:24.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:24.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:24.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:25.099 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:25.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:25.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:25.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:25.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:25.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:25.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:25.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:25.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:25.686 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:25.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 28725 28709 179.399 153.375 0.00608062 0.0055686 2024-09-14T04:04:25.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:25.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:25.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:25.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:25.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:25.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:26.068 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:26.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:26.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:26.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:26.243 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:26.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:26.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:26.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:26.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:26.621 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:26.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:26 smithi001 ceph-mon[107399]: pgmap v465: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:04:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:26 smithi001 ceph-mon[112165]: pgmap v465: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:04:26.758 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:26.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 31107 31091 176.623 148.875 0.008842 0.00565473 2024-09-14T04:04:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:26.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:26 smithi012 ceph-mon[101180]: pgmap v465: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:04:26.897 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:26.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:26.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:26] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-14T04:04:27.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:27.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:27.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:27.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:27.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:27.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:27.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:27.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:27.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:27.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:27.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:27.754 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:27.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 33227 33211 172.946 132.5 0.00613071 0.00577668 2024-09-14T04:04:27.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:27.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:27.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:27.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:28.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:28.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:28.178 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:28.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:28.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:28.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:28.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:28.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:28 smithi001 ceph-mon[112165]: pgmap v466: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-14T04:04:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:28 smithi001 ceph-mon[107399]: pgmap v466: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-14T04:04:28.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:28.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:28.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:28.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:28.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:28.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 35246 35230 169.348 126.188 0.00558917 0.00589944 2024-09-14T04:04:28.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:28 smithi012 ceph-mon[101180]: pgmap v466: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-09-14T04:04:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:28.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:28.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:28.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:29.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:29.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:29.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:29.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:29.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:29.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:29.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:29.534 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:29.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:29.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:29.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:29.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 36991 36975 165.041 109.062 0.00413724 0.00605358 2024-09-14T04:04:29.810 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:29.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:29.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:29.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:30.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:30.088 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:30.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:30.217 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:30.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:30.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:30.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:30.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:30.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:30.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:30 smithi001 ceph-mon[107399]: pgmap v467: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-14T04:04:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:30 smithi001 ceph-mon[112165]: pgmap v467: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-14T04:04:30.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:30.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:30.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 38965 38949 162.264 123.375 0.00207198 0.0061541 2024-09-14T04:04:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:30 smithi012 ceph-mon[101180]: pgmap v467: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-14T04:04:30.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:30.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:30.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:30.975 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:31.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:31.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:31.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:31.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:31.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:31.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:31.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:31.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:31.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:31.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:31.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:31.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 40841 40825 159.45 117.25 0.00765824 0.00626608 2024-09-14T04:04:31.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:31.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:31.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:31.993 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:31.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:32.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:32.121 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:32.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:32.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:32.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:32.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:32.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:32.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:32.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:32.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:32.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:32.706 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:32 smithi001 ceph-mon[107399]: pgmap v468: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-14T04:04:32.707 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:32 smithi001 ceph-mon[112165]: pgmap v468: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-14T04:04:32.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 42837 42821 157.407 124.75 0.00739879 0.00634815 2024-09-14T04:04:32.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:32.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:32 smithi012 ceph-mon[101180]: pgmap v468: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-14T04:04:32.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:32.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:32.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:33.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:33.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:33.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:33.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:33.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:33.378 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:04:33.378 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:04:33.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:33.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:33.550 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:33.652 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:33.689 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:33.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 44791 44775 155.446 122.125 0.00393547 0.00642805 2024-09-14T04:04:33.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:33.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:33.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:33.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:33.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:34.005 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:34.074 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:34.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:34.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:34.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:34.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:34.395 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:34.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:34.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:04:34.540 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:34.641 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:34 smithi001 ceph-mon[107399]: pgmap v469: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:34.642 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:34 smithi001 ceph-mon[112165]: pgmap v469: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:34.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:34.647 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:34 smithi012 ceph-mon[101180]: pgmap v469: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:34.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:34.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:34.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 46671 46655 153.446 117.5 0.00964626 0.00651213 2024-09-14T04:04:34.788 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.849 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:34.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:34.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:34.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:34.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:34.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:35.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:35.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:35.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:35.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:35.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:35.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:35.414 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:04:35.414 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:04:35.458 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:04:35.459 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:04:35.493 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:04:35.493 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:04:35.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:35.541 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:04:35.542 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:04:35.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:35.580 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:04:35.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:35.581 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3657008417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:35.581 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:35.581 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3657008417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:35.581 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:04:35.620 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:04:35.621 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:04:35.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:35.653 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:04:35.653 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:04:35.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:35.691 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:04:35.691 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:04:35.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:35.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:04:35.782730+0000 min lat: 0.00124478 max lat: 0.131492 avg lat: 0.00661336 2024-09-14T04:04:35.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:04:35.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 48376 48360 151.102 106.562 0.0106202 0.00661336 2024-09-14T04:04:35.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:35.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3657008417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:35.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:35.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:35.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:36.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:36.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:36.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:36.243 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:36.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:36.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:36.488 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:36.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:36.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:36.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:36.643 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:36 smithi001 ceph-mon[107399]: pgmap v470: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:36.643 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:36 smithi001 ceph-mon[112165]: pgmap v470: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:36.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:36.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:36.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 50059 50043 148.915 105.188 0.00620202 0.00670902 2024-09-14T04:04:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:36.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:36 smithi012 ceph-mon[101180]: pgmap v470: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:04:36.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:36.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:36.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:36] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-14T04:04:37.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:37.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:37.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.222 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:37.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:37.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:37.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:37.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:37.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:37.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:37.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:37.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:37.641 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:37.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:37.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:37.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:37.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 15 51739 51724 146.921 105.062 0.00513583 0.00680074 2024-09-14T04:04:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:37.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:37.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:37.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:38.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:38.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:38.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:38.224 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:38.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:38.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:38.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:38.427 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-14T04:04:38.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:38.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:38.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.655 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:38.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:38.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:38 smithi001 ceph-mon[107399]: pgmap v471: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.54k op/s 2024-09-14T04:04:38.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:38 smithi001 ceph-mon[112165]: pgmap v471: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.54k op/s 2024-09-14T04:04:38.756 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:38.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 53412 53396 145.076 104.5 0.013395 0.00688767 2024-09-14T04:04:38.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:38.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:38 smithi012 ceph-mon[101180]: pgmap v471: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.54k op/s 2024-09-14T04:04:38.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:38.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:38.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:39.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:39.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:39.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:39.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:39.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:39.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:39.354 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:39.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.507 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:39.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:39.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:39.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:39.730 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:39.734 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:39.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:39.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 54998 54982 143.161 99.125 0.0170892 0.00697819 2024-09-14T04:04:39.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:39.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:39.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:39.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:40.078 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:40.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:40.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:40.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:40.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:40.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:40.414 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:04:40.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:40.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:40.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:40.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[112165]: pgmap v472: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-14T04:04:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1217636535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[107399]: pgmap v472: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-14T04:04:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1217636535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:40.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 56552 56536 141.31 97.125 0.0116972 0.00707091 2024-09-14T04:04:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.812 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:40 smithi012 ceph-mon[101180]: pgmap v472: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-14T04:04:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1217636535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:04:40.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:40.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:40.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:41.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:41.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:41.199 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:41.240 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:41.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:41.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:41.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:04:41.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:41.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:41.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:41.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:41.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[112165]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:04:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:41.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:41 smithi001 ceph-mon[107399]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T04:04:41.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:41.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 58153 58137 139.723 100.062 0.00500982 0.0071525 2024-09-14T04:04:41.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:41 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:04:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:41 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:04:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:04:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:41 smithi012 ceph-mon[101180]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T04:04:41.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:41.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:41.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:42.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:42.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:42.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:42.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:42.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:42.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.368 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:42.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.522 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:42.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:42.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:42 smithi001 ceph-mon[107399]: pgmap v474: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:04:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:42 smithi001 ceph-mon[112165]: pgmap v474: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:04:42.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:42.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 59665 59649 138.048 94.5 0.00627555 0.00723896 2024-09-14T04:04:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:42 smithi012 ceph-mon[101180]: pgmap v474: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:04:42.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:42.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:42.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:43.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:43.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:43.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:43.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:43.223 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:43.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:43.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:43.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:43.503 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:43.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:43.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:43.648 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:43.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:43.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:43.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 61048 61032 136.205 86.4375 0.00432837 0.0073367 2024-09-14T04:04:43.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:43.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:43.921 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.922 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.922 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:43.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:43.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:44.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:44.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:44.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:44.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:44.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:44.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:44.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:44.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:44.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:44.629 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:44.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:44.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:44.782 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:44 smithi001 ceph-mon[107399]: pgmap v475: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:04:44.782 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:44 smithi001 ceph-mon[112165]: pgmap v475: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:04:44.787 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 62595 62579 134.835 96.6875 0.00724314 0.00741189 2024-09-14T04:04:44.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:44.906 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:44.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:45.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:45.050 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:44 smithi012 ceph-mon[101180]: pgmap v475: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:04:45.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:45.193 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:45.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:45.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:45.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:45.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:45.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:45.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:45.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:45.748 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:45.787 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 64199 64183 133.682 100.25 0.0118618 0.00747596 2024-09-14T04:04:45.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:45.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:45.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:45.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:46.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:46.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:46.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:46.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:46.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:46.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:46.454 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:46.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:46.566 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:04:46.588 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:46.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:46.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:46.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:46 smithi001 ceph-mon[112165]: pgmap v476: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:04:46.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:46 smithi001 ceph-mon[107399]: pgmap v476: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:04:46.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:46] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-14T04:04:46.787 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 65685 65669 132.365 92.875 0.0100354 0.00754909 2024-09-14T04:04:46.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:46.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:46.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:46.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:47.012 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89043 8.a 8.216a668a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.11143172999999, 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.002694494, 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.013 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89043 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:04:47.014 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632795+0000", 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.99e-06 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632924+0000", 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012871200000000001 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632937+0000", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2955999999999999e-05 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632948+0000", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1657999999999999e-05 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635088+0000", 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002139312 2024-09-14T04:04:47.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635096+0000", 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5199999999999997e-06 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658069+0000", 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022972731999999999 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.017 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658078+0000", 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7579999999999998e-06 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635476+0000", 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97739821800000004 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.018 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635488+0000", 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1637999999999999e-05 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89049 8.3 8.20f81863 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10921886599999, 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000897207, 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.019 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89049 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.020 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635007+0000", 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.342e-06 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635067+0000", 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0242999999999999e-05 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635079+0000", 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1970999999999999e-05 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.022 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635088+0000", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9269999999999994e-06 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.650192+0000", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015103836000000001 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.650202+0000", 2024-09-14T04:04:47.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.679e-06 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635892+0000", 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98569037199999998 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635903+0000", 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0837e-05 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.024 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89050 8.3 8.612cea13 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10911729700001, 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0008373749999999, 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89050 2024-09-14T04:04:47.025 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635108+0000", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3499999999999996e-07 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635163+0000", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4635000000000001e-05 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635172+0000", 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6630000000000005e-06 2024-09-14T04:04:47.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.028 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635180+0000", 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5320000000000001e-06 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653159+0000", 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017979311000000001 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.029 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653172+0000", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2466999999999999e-05 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635935+0000", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98276317899999999 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635945+0000", 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9529999999999994e-06 2024-09-14T04:04:47.030 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89058 8.3 8.951b07a3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10903491699997, 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000785891, 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.031 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89058 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:04:47.032 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8599999999999998e-07 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635240+0000", 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9608999999999999e-05 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.033 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635248+0000", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1710000000000002e-06 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635254+0000", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0059999999999998e-06 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653236+0000", 2024-09-14T04:04:47.034 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017982206000000001 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653245+0000", 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4889999999999995e-06 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635969+0000", 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98272394100000005 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.035 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635976+0000", 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.883e-06 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89059 8.a 8.d6fe050a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635263+0000", 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10896163400002, 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0002887629999999, 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.036 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89059 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635263+0000", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.037 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635263+0000", 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635264+0000", 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2900000000000004e-07 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635310+0000", 2024-09-14T04:04:47.038 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6301000000000003e-05 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635317+0000", 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.888e-06 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635322+0000", 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4770000000000002e-06 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.039 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658752+0000", 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.023429219000000001 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658760+0000", 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3760000000000008e-06 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635544+0000", 2024-09-14T04:04:47.040 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97678361700000005 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635552+0000", 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3559999999999993e-06 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89060 8.3 8.edcc3963 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635337+0000", 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10888819299998, 2024-09-14T04:04:47.041 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0006675229999999, 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89060 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635337+0000", 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.042 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635337+0000", 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635337+0000", 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7599999999999997e-07 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.043 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635388+0000", 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1198e-05 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635397+0000", 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6230000000000003e-06 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635404+0000", 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0260000000000002e-06 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.044 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653278+0000", 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017874533000000001 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653286+0000", 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3359999999999997e-06 2024-09-14T04:04:47.045 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635997+0000", 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98271169000000003 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.046 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.636004+0000", 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6409999999999996e-06 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.047 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89061 8.3 8.ab84b7b3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10879924800003, 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000835736, 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.048 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89061 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.049 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0399999999999996e-07 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635477+0000", 2024-09-14T04:04:47.050 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0652999999999998e-05 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635484+0000", 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5020000000000003e-06 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635489+0000", 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2129999999999999e-06 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.051 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653317+0000", 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017827393 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653325+0000", 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8499999999999994e-06 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.636247+0000", 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98292271799999997 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.052 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.636261+0000", 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3903000000000001e-05 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89068 8.7 8.e11c6ed7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635497+0000", 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.10872729099998, 2024-09-14T04:04:47.053 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99695908499999997, 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89068 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.054 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635497+0000", 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635497+0000", 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635498+0000", 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0399999999999995e-07 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.055 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635549+0000", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1168000000000002e-05 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635558+0000", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6719999999999993e-06 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635564+0000", 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.857e-06 2024-09-14T04:04:47.056 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.636690+0000", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0011263670000000001 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.636715+0000", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4655e-05 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658080+0000", 2024-09-14T04:04:47.057 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021364782999999998 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658087+0000", 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2099999999999996e-06 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.632367+0000", 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97427988499999996 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.058 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.632457+0000", 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9784000000000006e-05 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89073 8.1 8.4f1937c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.639766+0000", 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 412.104458534, 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99436855099999999, 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.059 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89073 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639766+0000", 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.060 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639766+0000", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639768+0000", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.575e-06 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639820+0000", 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1727999999999997e-05 2024-09-14T04:04:47.061 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639840+0000", 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0029000000000001e-05 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.639850+0000", 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0688e-05 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.062 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.648968+0000", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0091173629999999999 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.648977+0000", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5009999999999993e-06 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634111+0000", 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98513398799999996 2024-09-14T04:04:47.063 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.634135+0000", 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3679000000000001e-05 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:121341 8.3 8:c72b1ef3:::benchmark_data_smithi001_125658_object121340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 374.48904932200003, 2024-09-14T04:04:47.064 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.014175067, 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 121341 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.065 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255177+0000", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.012e-06 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255204+0000", 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6996999999999999e-05 2024-09-14T04:04:47.066 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255218+0000", 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3212e-05 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255228+0000", 2024-09-14T04:04:47.067 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9399999999999997e-06 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.259603+0000", 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043758759999999999 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.259626+0000", 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.211e-05 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.068 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.268869+0000", 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0092438880000000005 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.268879+0000", 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7140000000000005e-06 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257381+0000", 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98850154899999998 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.069 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257528+0000", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014704900000000001 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257698+0000", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000169789 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.266660+0000", 2024-09-14T04:04:47.070 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089624390000000009 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269181+0000", 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002520834 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269280+0000", 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9536000000000004e-05 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.072 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269321+0000", 2024-09-14T04:04:47.072 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0234999999999997e-05 2024-09-14T04:04:47.072 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.072 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.072 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.073 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269351+0000", 2024-09-14T04:04:47.073 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9887e-05 2024-09-14T04:04:47.073 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.073 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.073 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144668 8.a 8:53b134be:::benchmark_data_smithi001_125658_object19429:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 343.56493811000001, 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0089713979999999, 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.074 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144668 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.075 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179289+0000", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8920000000000001e-06 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179295+0000", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4099999999999996e-06 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.076 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179427+0000", 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013198600000000001 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179438+0000", 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1477999999999999e-05 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179923+0000", 2024-09-14T04:04:47.077 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048479699999999997 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179935+0000", 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1929999999999999e-05 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188185+0000", 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0082499250000001 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.078 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188258+0000", 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2979999999999996e-05 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144680 8.a 8:5ac33e5e:::benchmark_data_smithi001_125658_object19441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 343.56324073000002, 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.024066712, 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.079 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144680 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:04:47.080 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5200000000000002e-07 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180987+0000", 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1730000000000002e-06 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.081 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180993+0000", 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1990000000000003e-06 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180998+0000", 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8840000000000002e-06 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.082 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.083 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.181072+0000", 2024-09-14T04:04:47.083 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4634000000000004e-05 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.181081+0000", 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2109999999999998e-06 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188740+0000", 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.007659597 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.084 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188893+0000", 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015260800000000001 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.189452+0000", 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055930699999999999 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204842+0000", 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015390019 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204883+0000", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0751999999999999e-05 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204945+0000", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1816000000000006e-05 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205003+0000", 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8137000000000003e-05 2024-09-14T04:04:47.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205051+0000", 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7923000000000002e-05 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144687 8.a 8:5927234f:::benchmark_data_smithi001_125658_object19448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:04:47.087 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 343.55561113800002, 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0239874259999999, 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144687 2024-09-14T04:04:47.088 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.089 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188615+0000", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.747e-06 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188620+0000", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4150000000000001e-06 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188635+0000", 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5534999999999999e-05 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.090 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188643+0000", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2910000000000002e-06 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188870+0000", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000227631 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188946+0000", 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5879999999999999e-05 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.091 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.201935+0000", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0129886370000001 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.202067+0000", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013186900000000001 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.202199+0000", 2024-09-14T04:04:47.092 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013186900000000001 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212260+0000", 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010061614 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212431+0000", 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017089000000000001 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.093 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212503+0000", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2368999999999998e-05 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212558+0000", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4379999999999998e-05 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212601+0000", 2024-09-14T04:04:47.094 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3299e-05 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144694 8.a 8:5329308c:::benchmark_data_smithi001_125658_object19455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 343.54622615, 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0241956910000001, 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.095 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144694 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.096 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198000+0000", 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.813e-06 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198007+0000", 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.421e-06 2024-09-14T04:04:47.097 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198022+0000", 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5515e-05 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198030+0000", 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9820000000000008e-06 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.098 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.199399+0000", 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0013681640000000001 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.199424+0000", 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5190000000000001e-05 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.099 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.100 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.100 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205099+0000", 2024-09-14T04:04:47.100 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0056756440000001 2024-09-14T04:04:47.100 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.100 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205233+0000", 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000133165 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205364+0000", 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131347 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.101 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212626+0000", 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0072621350000000003 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222029+0000", 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0094028069999999991 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222130+0000", 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010163299999999999 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.102 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222166+0000", 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5932000000000001e-05 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222194+0000", 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7943e-05 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172983 8.1 8:897d1811:::benchmark_data_smithi001_125658_object47744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:04:47.103 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 308.21633973799999, 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.003049342, 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172983 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.104 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527887+0000", 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5600000000000001e-06 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.105 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527891+0000", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1359999999999999e-06 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527907+0000", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6665000000000001e-05 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527914+0000", 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0960000000000003e-06 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.106 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532469+0000", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045540349999999997 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532478+0000", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5400000000000001e-06 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523129+0000", 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99065059200000005 2024-09-14T04:04:47.107 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523243+0000", 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114265 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523315+0000", 2024-09-14T04:04:47.108 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2384000000000004e-05 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530751+0000", 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0074354770000000002 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530769+0000", 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8235000000000001e-05 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.109 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530820+0000", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1110999999999999e-05 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530888+0000", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7537000000000001e-05 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530934+0000", 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6709000000000003e-05 2024-09-14T04:04:47.110 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172987 8.1 8:82557b84:::benchmark_data_smithi001_125658_object47748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 308.21345433800002, 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.007018996, 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.111 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172987 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.112 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530772+0000", 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.739e-06 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.113 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530775+0000", 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.191e-06 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530785+0000", 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4669999999999993e-06 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530866+0000", 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1584999999999999e-05 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.114 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532502+0000", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001635818 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532508+0000", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.908e-06 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527152+0000", 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99464408800000004 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.115 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527289+0000", 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013704 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527406+0000", 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116469 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.116 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530983+0000", 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035769510000000001 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537637+0000", 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0066542190000000003 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.117 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537686+0000", 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8838999999999998e-05 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537741+0000", 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5473000000000002e-05 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.118 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537789+0000", 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8208999999999999e-05 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172997 8.1 8:86774ea3:::benchmark_data_smithi001_125658_object47758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 308.202244501, 2024-09-14T04:04:47.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0038687230000001, 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172997 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.120 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541981+0000", 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.221e-06 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541986+0000", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.464e-06 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542001+0000", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4718e-05 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542008+0000", 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.559e-06 2024-09-14T04:04:47.122 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542161+0000", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000152905 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542187+0000", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5644e-05 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531053+0000", 2024-09-14T04:04:47.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98886591700000004 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531180+0000", 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012756700000000001 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531305+0000", 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012458099999999999 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.125 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545639+0000", 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014334219 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545657+0000", 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7595999999999999e-05 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545736+0000", 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9217999999999999e-05 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545789+0000", 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3072000000000003e-05 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545849+0000", 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0041999999999999e-05 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.127 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:173000 8.1 8:87fd81a4:::benchmark_data_smithi001_125658_object47761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 308.19995884500003, 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.001832101, 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 173000 2024-09-14T04:04:47.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544267+0000", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.063e-06 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544270+0000", 2024-09-14T04:04:47.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1200000000000002e-06 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544279+0000", 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3049999999999998e-06 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544285+0000", 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5729999999999998e-06 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544354+0000", 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9206000000000001e-05 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544363+0000", 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1050000000000001e-06 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534232+0000", 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98986859599999999 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.132 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534331+0000", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9159000000000004e-05 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534400+0000", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9410000000000001e-05 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545927+0000", 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01152623 2024-09-14T04:04:47.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545941+0000", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4076e-05 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545982+0000", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0939000000000002e-05 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.546040+0000", 2024-09-14T04:04:47.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8448e-05 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.546098+0000", 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7871000000000002e-05 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249045 8.7 8:e36474f6:::benchmark_data_smithi001_125658_object123806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:04:47.135 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 214.39992327900001, 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0581002930000001, 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249045 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.136 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344303+0000", 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4959999999999999e-06 2024-09-14T04:04:47.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344307+0000", 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0400000000000003e-06 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344317+0000", 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5759999999999995e-06 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344324+0000", 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7819999999999994e-06 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344858+0000", 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00053357000000000003 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.139 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344868+0000", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6439999999999995e-06 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376162+0000", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0312939839999999 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376249+0000", 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7023999999999999e-05 2024-09-14T04:04:47.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376790+0000", 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054109600000000002 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387731+0000", 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010941683000000001 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402294+0000", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014562354 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402334+0000", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0154000000000002e-05 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402368+0000", 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.417e-05 2024-09-14T04:04:47.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402402+0000", 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3720000000000002e-05 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249083 8.7 8:e6756ecc:::benchmark_data_smithi001_125658_object123844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 214.358281361, 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016715622, 2024-09-14T04:04:47.143 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249083 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.144 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385944+0000", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5899999999999996e-07 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385947+0000", 2024-09-14T04:04:47.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.349e-06 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385959+0000", 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1462e-05 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385965+0000", 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1870000000000002e-06 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386615+0000", 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00064992000000000001 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386640+0000", 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5106e-05 2024-09-14T04:04:47.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387765+0000", 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0011247430000001 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387888+0000", 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123702 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:04:47.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387975+0000", 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6299999999999997e-05 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.394473+0000", 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064983289999999997 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402469+0000", 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079956669999999997 2024-09-14T04:04:47.149 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402562+0000", 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3187000000000001e-05 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402616+0000", 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4555999999999997e-05 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:04:47.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402659+0000", 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2555000000000002e-05 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:04:47.151 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:04:47.152 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:46 smithi012 ceph-mon[101180]: pgmap v476: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:04:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.153 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:47.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:47.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:47.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:47.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:47.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:47.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:47.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:47.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.706 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:47.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:47.787 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 67208 67192 131.204 95.1875 0.0133601 0.00761647 2024-09-14T04:04:47.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:47.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:47.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:47.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:47.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:48.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:48.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:48.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:48.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:48.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:48.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:48.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:48.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:48.415 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:48.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:48.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:48.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:04:48.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:48.693 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:48.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 68617 68601 129.895 88.0625 0.0106107 0.00769319 2024-09-14T04:04:48.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:48.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:48.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:48.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:48.974 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:48 smithi001 ceph-mon[107399]: pgmap v477: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:04:48.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:48 smithi001 ceph-mon[112165]: pgmap v477: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:04:48.976 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:48.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:49.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:48 smithi012 ceph-mon[101180]: pgmap v477: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:04:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.118 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:49.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:49.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:49.259 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:49.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-14T04:04:49.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:49.364 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:49.385 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:49.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:49.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:49.491 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:49.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:49.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:49.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:49.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 69953 69937 128.529 83.5 0.0127681 0.00777496 2024-09-14T04:04:49.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:49.834 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:49.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:49.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:49.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:50.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.107 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:50.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:50.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.243 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:50.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:50.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:50.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:50.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:50.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:50.522 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:50.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:50.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:50.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:50.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:50.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 71399 71383 127.439 90.375 0.0166188 0.00784197 2024-09-14T04:04:50.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:50.914 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:50.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:50.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:51.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:51.125 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:51.185 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[107399]: pgmap v478: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-14T04:04:51.185 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:51.185 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:51.185 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[112165]: pgmap v478: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-14T04:04:51.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:51.186 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:51.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:51.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:50 smithi012 ceph-mon[101180]: pgmap v478: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-14T04:04:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:04:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:51.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:51.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:51.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:51.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:51.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:51.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:51.730 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:51.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:51.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 72718 72702 126.188 82.4375 0.0141074 0.00791767 2024-09-14T04:04:51.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:51.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:51.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:51.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:52.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:52.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:52.107 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:04:52.107 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:04:52.142 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:04:52.143 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:04:52.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:52.184 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:04:52.185 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:04:52.220 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:04:52.220 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:04:52.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:52.253 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:04:52.253 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:04:52.287 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:04:52.287 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:04:52.321 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:04:52.321 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:04:52.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:52.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:52.361 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:04:52.361 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:04:52.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:52.499 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:52.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:52.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:52.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:52.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:52.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 73935 73919 124.833 76.0625 0.0127193 0.00800611 2024-09-14T04:04:52.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:52.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:52.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:52.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:53.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:53.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:53.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:52 smithi001 ceph-mon[107399]: pgmap v479: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:04:53.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3449362674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:53.094 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:52 smithi001 ceph-mon[112165]: pgmap v479: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:04:53.094 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3449362674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:53.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:53.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:53.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:52 smithi012 ceph-mon[101180]: pgmap v479: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:04:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3449362674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:04:53.354 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:53.456 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:53.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.498 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:53.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:53.599 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:53.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:53.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:53.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:53.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:53.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 75261 75245 123.729 82.875 0.00992472 0.00807733 2024-09-14T04:04:53.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:53.899 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:53.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:54.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:54.039 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:54.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:54.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:54.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:54.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:54.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:54.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:54.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:54.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:54.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:54.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 76600 76584 122.702 83.6875 0.00986204 0.00814432 2024-09-14T04:04:54.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:54.862 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:54.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:54.978 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:54 smithi001 ceph-mon[112165]: pgmap v480: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:54.978 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:54 smithi001 ceph-mon[107399]: pgmap v480: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:54.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:54.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:55.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:55.133 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:55.235 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:55.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:55.346 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:04:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=sqlstore.transactions t=2024-09-14T04:04:55.170432122Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T04:04:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:54 smithi012 ceph-mon[101180]: pgmap v480: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:55.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:55.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:55.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:55.557 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:55.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:55.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:55.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:04:55.788804+0000 min lat: 0.00124478 max lat: 0.131492 avg lat: 0.00821009 2024-09-14T04:04:55.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:04:55.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 77941 77925 121.73 83.8125 0.00469777 0.00821009 2024-09-14T04:04:55.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:55.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:55.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:55.945 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:04:55.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:55.995 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:56.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:56.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:56.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3793101166' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3793101166' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:56.272 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:56.336 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3793101166' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:04:56.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:56.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:56.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:04:56.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:56.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:56.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:04:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.695 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:56.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:04:56.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 79245 79229 120.739 81.5 0.00350506 0.00827615 2024-09-14T04:04:56.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:56.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:56.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:56.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:56.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[112165]: pgmap v481: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[112165]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[107399]: pgmap v481: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:04:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-mon[107399]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T04:04:56.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:04:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:04:56] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-14T04:04:57.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:57.104 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:57.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:57.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:56 smithi012 ceph-mon[101180]: pgmap v481: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:04:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:56 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:04:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:56 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:04:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:56 smithi012 ceph-mon[101180]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T04:04:57.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:57.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:57.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:57.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:57.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:57.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:57.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:04:57.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:04:57.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 80547 80531 119.802 81.375 0.015318 0.00834249 2024-09-14T04:04:57.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:57.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:57.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:58.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:58.061 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:58.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:58.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:58.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:58.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:58.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:58.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:58.478 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:58.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:58.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:58.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:58.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:04:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:58.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:04:58.784 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:04:58.784 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:04:58.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 81913 81897 119.001 85.375 0.0116315 0.00839847 2024-09-14T04:04:58.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:58.894 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:58.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:58.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:59.035 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:59.043 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:04:58 smithi001 ceph-mon[107399]: pgmap v483: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:59.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:04:58 smithi001 ceph-mon[112165]: pgmap v483: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:59.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:04:59.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:04:59.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:04:59.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:04:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:04:58 smithi012 ceph-mon[101180]: pgmap v483: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T04:04:59.350 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:04:59.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.436 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:59.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:04:59.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:04:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:59.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:04:59.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:59.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:59.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 83056 83040 117.92 71.4375 0.0166106 0.00847551 2024-09-14T04:04:59.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:04:59.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:04:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:04:59.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:04:59.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:04:59.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:04:59.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:00.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:00.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:00.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:00.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:00.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:00.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:00.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:00.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:00.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:00.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:00.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 84246 84230 116.952 74.375 0.00596206 0.00854476 2024-09-14T04:05:00.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:00.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:00.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:00.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:01.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:01.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:01.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:01.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:00 smithi001 ceph-mon[107399]: pgmap v484: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:01.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:00 smithi001 ceph-mon[112165]: pgmap v484: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:01.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:00 smithi012 ceph-mon[101180]: pgmap v484: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:01.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:01.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:01.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:01.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:01.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:01.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:01.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:01.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 85434 85418 116.024 74.25 0.00544865 0.00861407 2024-09-14T04:05:01.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:01.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:01.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:02.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:02.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:02.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:02.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:02.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:02.354 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:02.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:02.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:02.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:02.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:02.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:02.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.759 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:02.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 86564 86548 115.058 70.625 0.00495282 0.00868424 2024-09-14T04:05:02.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:02.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:02.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:02.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:03.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:03.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:03.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:03.161 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:02 smithi001 ceph-mon[107399]: pgmap v485: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:05:03.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:02 smithi001 ceph-mon[112165]: pgmap v485: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:05:03.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:03.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:02 smithi012 ceph-mon[101180]: pgmap v485: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:05:03.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:03.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:03.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:03.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:03.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:03.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:03.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 87648 87632 114.073 67.75 0.0143587 0.0087613 2024-09-14T04:05:03.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:03.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:03.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:03.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:03.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:04.098 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:04.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:04.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:04.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:04.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:04.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:04.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:04.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:04.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:04.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:04.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:04.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 88751 88735 113.152 68.9375 0.0133622 0.00883273 2024-09-14T04:05:04.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:04.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:04.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:04.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:04.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:05.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:05.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:05.142 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[107399]: pgmap v486: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:05.142 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:05.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:05.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[112165]: pgmap v486: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:05.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:05.143 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:05.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:05:05.214 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:05.276 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:04 smithi012 ceph-mon[101180]: pgmap v486: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:05.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:05.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:05.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:05.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:05.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:05.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-14T04:05:05.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:05.748 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:05.795 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 89932 89916 112.359 73.8125 0.0132175 0.00889402 2024-09-14T04:05:05.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:05.826 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:05.846 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:05.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:05.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:05.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:05.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:05.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:06.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:06.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:06.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:06.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:06.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:06.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:06.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:06.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:06.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:06.700 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:06.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 91059 91043 111.538 70.4375 0.00882249 0.00896017 2024-09-14T04:05:06.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:06.814 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:06] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-14T04:05:06.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:06.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:06.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:06.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:06.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:07.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.149 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:07.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:06 smithi001 ceph-mon[112165]: pgmap v487: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:05:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:06 smithi001 ceph-mon[107399]: pgmap v487: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:05:07.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:07.287 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:07.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:07.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:06 smithi012 ceph-mon[101180]: pgmap v487: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:05:07.389 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:07.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:07.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:07.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:07.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:07.667 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:07.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:07.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 92274 92258 110.853 75.9375 0.0076243 0.00901617 2024-09-14T04:05:07.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:07.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:07.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:07.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:07.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:07.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:08.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:08.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:08.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:08.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:08.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:08.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:08.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:08.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:08.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:08.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:08.674 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:08.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:08.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 93518 93502 110.228 77.75 0.0148587 0.00906679 2024-09-14T04:05:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:08.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:08.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:08.951 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:08.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:09.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.092 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:09.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:09.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:09.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:08 smithi001 ceph-mon[107399]: pgmap v488: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T04:05:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:08 smithi001 ceph-mon[112165]: pgmap v488: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T04:05:09.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:09.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:08 smithi012 ceph-mon[101180]: pgmap v488: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-14T04:05:09.372 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:09.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:09.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:09.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:09.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:09.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:09.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:09.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:09.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:09.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 15 94768 94753 109.635 78.1875 0.00697987 0.00911613 2024-09-14T04:05:09.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:09.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:09.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:10.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:10.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:10.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:10.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:10.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:10.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:10.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:10.499 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:10.601 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:10.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:10.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:10.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:10.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 95975 95959 109.012 75.375 0.00964313 0.0091678 2024-09-14T04:05:10.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:10.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:10.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:10.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:11.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:11.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:11.153 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:10 smithi001 ceph-mon[107399]: pgmap v489: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:11.154 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:10 smithi001 ceph-mon[112165]: pgmap v489: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:11.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:11.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:11.240 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:05:11.240 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:05:11.288 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:05:11.288 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:05:11.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:11.334 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:05:11.335 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:05:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:10 smithi012 ceph-mon[101180]: pgmap v489: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:11.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:11.375 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:05:11.376 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:05:11.438 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:05:11.438 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:05:11.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:11.485 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:05:11.485 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:05:11.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:11.516 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:05:11.516 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:05:11.532 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:05:11.533 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:05:11.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:11.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:11.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:11.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:11.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 97013 96997 108.224 64.875 0.0142095 0.00923482 2024-09-14T04:05:11.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:11.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:11.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:12.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:12.047 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:05:12.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:12.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:12.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:12.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/755451831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:11 smithi012 ceph-mon[101180]: pgmap v490: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T04:05:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3878245956' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:11 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:12.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/755451831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[107399]: pgmap v490: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3878245956' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/755451831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:12.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[112165]: pgmap v490: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-14T04:05:12.408 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3878245956' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.408 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:11 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:12.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:12.507 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:12.579 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:05:12.579 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:05:12.580 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:05:12.580 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:05:12.580 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:05:12.609 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.650 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:12.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:12.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:12.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:12.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:12.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 98191 98175 107.616 73.625 0.0122363 0.00928714 2024-09-14T04:05:12.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:12.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:12.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:13.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[112165]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:13.102 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:12 smithi001 ceph-mon[107399]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T04:05:13.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:13.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:12 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:12 smithi012 ceph-mon[101180]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T04:05:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:13.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:13.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:13.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:13.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:13.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:13.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:13.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 99357 99341 107.017 72.875 0.010799 0.00933951 2024-09-14T04:05:13.897 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:13.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:13.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:13.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:14.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:14.078 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:14.080 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:13 smithi001 ceph-mon[112165]: pgmap v492: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:05:14.080 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:13 smithi001 ceph-mon[107399]: pgmap v492: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:05:14.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:05:14.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:14.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:14.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:13 smithi012 ceph-mon[101180]: pgmap v492: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:05:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.368 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:14.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:14.469 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:14.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:14.571 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:14.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:14.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:14.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:14.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:14.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 100479 100463 106.39 70.125 0.0102335 0.00939462 2024-09-14T04:05:14.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:14.959 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:14.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:15.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:15.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:15.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:15.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:15.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:15.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:15.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:05:15.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:15.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:15.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:15.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:15.787 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:15.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:05:15.797925+0000 min lat: 0.00124478 max lat: 0.175154 avg lat: 0.0094575 2024-09-14T04:05:15.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:05:15.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 101504 101488 105.684 64.0625 0.00720498 0.0094575 2024-09-14T04:05:15.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:15.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: pgmap v493: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: pgmap v493: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.931 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: pgmap v493: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T04:05:15.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:15.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:15.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:15.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:15.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:15.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:15.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:16.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:16.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:16.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:16.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:16.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:16.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:16.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:16.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:16.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:16.649 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:16.695 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:16.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:16.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 102648 102632 105.124 71.5 0.0172721 0.00950806 2024-09-14T04:05:16.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:16.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:16.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:16] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-14T04:05:16.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:16.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:16.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:17.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:05:17.084 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:17.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:17.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:17.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:17.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:17.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:17.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:17.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:17.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:17.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:17.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:17.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 103693 103677 104.482 65.3125 0.00851557 0.00956631 2024-09-14T04:05:17.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:17.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:17.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:17.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:18.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:18.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:18.140 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:18.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:18.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:18.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:18.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:18.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:18.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:18.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:18.625 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:18.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:18.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:18 smithi001 ceph-mon[107399]: pgmap v494: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:18 smithi001 ceph-mon[112165]: pgmap v494: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:18.797 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:18.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 104768 104752 103.89 67.1875 0.00351464 0.00961286 2024-09-14T04:05:18.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:18.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:18 smithi012 ceph-mon[101180]: pgmap v494: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:18.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:18.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:18.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:19.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.095 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:19.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:19.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:19.238 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:19.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:19.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:19.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:19.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:19.617 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:19.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:19 smithi001 ceph-mon[107399]: pgmap v495: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:19 smithi001 ceph-mon[112165]: pgmap v495: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:19.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:19.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 105808 105792 103.282 65 0.0116467 0.00967729 2024-09-14T04:05:19.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:19 smithi012 ceph-mon[101180]: pgmap v495: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-14T04:05:19.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:19.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:19.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:20.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:20.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:20.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:20.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:20.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:20.353 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:20.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:20.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:20.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:20.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:20.653 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:20.653 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:20.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:20.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:20.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 106798 106782 102.645 61.875 0.00941751 0.00973737 2024-09-14T04:05:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:20.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:20.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:20.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:20.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:20.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:21.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:21.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:21.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:21.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:21.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:21.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:21.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:21.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:21.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:21 smithi001 ceph-mon[112165]: pgmap v496: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:05:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:21 smithi001 ceph-mon[107399]: pgmap v496: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:05:21.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:21.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:21.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:21.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:21.799 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 107757 107741 101.998 59.9375 0.00932877 0.00979899 2024-09-14T04:05:21.834 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:21 smithi012 ceph-mon[101180]: pgmap v496: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:05:21.872 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:21.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:21.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:22.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:22.021 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:22.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.153 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:22.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:22.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:22.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:22.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:22.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:22.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:22.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:22.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:22.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:22.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:22.799 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 108805 108789 101.454 65.5 0.0215588 0.00985217 2024-09-14T04:05:22.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:22.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:22.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:22.990 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:22.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:23.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.133 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:23.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:23.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:23.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:23.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:23.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:23.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.540 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:23.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:23.641 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:23.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:23.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:23 smithi001 ceph-mon[112165]: pgmap v497: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:05:23.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:23 smithi001 ceph-mon[107399]: pgmap v497: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:05:23.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:23.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 15 109874 109859 100.945 66.875 0.029615 0.00990194 2024-09-14T04:05:23.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:23.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:23 smithi012 ceph-mon[101180]: pgmap v497: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:05:23.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:23.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:23.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:24.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:24.104 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:24.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:24.230 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:05:24.230 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:05:24.230 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:70509 9.6 e141/135 9:6c0d05b1:::benchmark_data_smithi001_156540_object70508:head v 141'4323, mlcod=141'4319)", 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:50.182121+0000", 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 34.009196901999999, 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01975066, 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.231 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182121+0000", 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182121+0000", 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182122+0000", 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3799999999999996e-07 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182154+0000", 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1874e-05 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182172+0000", 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8224999999999999e-05 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182190+0000", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.736e-05 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182546+0000", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035645000000000002 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182592+0000", 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5324999999999997e-05 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.201793+0000", 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019201477000000002 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.201872+0000", 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9110999999999995e-05 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:70512 9.6 e141/135 9:692cd73d:::benchmark_data_smithi001_156540_object70511:head v 141'4324, mlcod=141'4319)", 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:50.182667+0000", 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 34.00865142, 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019232428999999999, 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182667+0000", 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182667+0000", 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182668+0000", 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2500000000000005e-07 2024-09-14T04:05:24.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182718+0000", 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9999e-05 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182729+0000", 2024-09-14T04:05:24.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0927999999999999e-05 2024-09-14T04:05:24.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.182734+0000", 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8239999999999996e-06 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.188712+0000", 2024-09-14T04:05:24.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059776960000000002 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.188766+0000", 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3664e-05 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.201877+0000", 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013111004000000001 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:50.201899+0000", 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2688999999999999e-05 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:72666 9.0 e141/135 9:0b9ebb5f:::benchmark_data_smithi001_156540_object72665:head v 141'4469, mlcod=141'4466)", 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:51.718909+0000", 2024-09-14T04:05:24.242 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.472409312000003, 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019693794000000001, 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718909+0000", 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718909+0000", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718910+0000", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.037e-06 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718958+0000", 2024-09-14T04:05:24.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7759000000000001e-05 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718975+0000", 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7224999999999999e-05 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.718984+0000", 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5050000000000007e-06 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.719186+0000", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020203199999999999 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.719247+0000", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1325e-05 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.738514+0000", 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019267268000000001 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:51.738603+0000", 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8642999999999998e-05 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:98298 9.0 e143/135 9:0f325d3a:::benchmark_data_smithi001_156540_object98297:head v 143'6077, mlcod=143'6075)", 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:12.898438+0000", 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 11.292880818, 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016551528999999999, 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.247 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898438+0000", 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898438+0000", 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898439+0000", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3799999999999996e-07 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898466+0000", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.762e-05 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898484+0000", 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7476e-05 2024-09-14T04:05:24.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898492+0000", 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1440000000000003e-06 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898642+0000", 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015019299999999999 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.898727+0000", 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4840999999999996e-05 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.914782+0000", 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016054826000000001 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:12.914989+0000", 2024-09-14T04:05:24.252 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000207491 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:99899 9.2 e143/135 9:42eb2e62:::benchmark_data_smithi001_156540_object99898:head v 143'6201, mlcod=143'6199)", 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:14.270844+0000", 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 9.9204748560000002, 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019804179000000002, 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.253 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270844+0000", 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270844+0000", 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270845+0000", 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1370000000000001e-06 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270874+0000", 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9332e-05 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270896+0000", 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2135999999999999e-05 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.270906+0000", 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0012e-05 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.271105+0000", 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000198975 2024-09-14T04:05:24.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.271154+0000", 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9035000000000001e-05 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.290595+0000", 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019440829999999999 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.290648+0000", 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2722000000000001e-05 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.258 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:99907 9.6 e143/135 9:603866cd:::benchmark_data_smithi001_156540_object99906:head v 143'6165, mlcod=143'6163)", 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:14.278833+0000", 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 9.9124852820000005, 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016427789000000002, 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278833+0000", 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278833+0000", 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278834+0000", 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0729999999999999e-06 2024-09-14T04:05:24.260 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278871+0000", 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7035e-05 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278884+0000", 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2465e-05 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278891+0000", 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5390000000000002e-06 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.279243+0000", 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035147999999999999 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.279331+0000", 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7973000000000006e-05 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.294821+0000", 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015490723 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.295261+0000", 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043950100000000002 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.263 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:99910 9.6 e143/135 9:6d173370:::benchmark_data_smithi001_156540_object99909:head v 143'6166, mlcod=143'6163)", 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:14.278913+0000", 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 9.9124051309999999, 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022746557000000001, 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278913+0000", 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.264 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278913+0000", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278914+0000", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.41e-07 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278942+0000", 2024-09-14T04:05:24.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8212999999999999e-05 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278948+0000", 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.472e-06 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.278953+0000", 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7679999999999999e-06 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.284601+0000", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056477749999999998 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.284658+0000", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7510999999999999e-05 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.301402+0000", 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016743551999999998 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.301660+0000", 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025792499999999999 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:99912 9.6 e143/135 9:6def5446:::benchmark_data_smithi001_156540_object99911:head v 143'6167, mlcod=143'6163)", 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:14.281596+0000", 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 9.9097220620000002, 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020265503000000001, 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.268 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281596+0000", 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281596+0000", 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281597+0000", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.043e-06 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281639+0000", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1128000000000003e-05 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281654+0000", 2024-09-14T04:05:24.270 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5936000000000001e-05 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.281662+0000", 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.401e-06 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.287965+0000", 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063034390000000001 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.288012+0000", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6762999999999999e-05 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.301663+0000", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013650944999999999 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.301862+0000", 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000198848 2024-09-14T04:05:24.272 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:99917 9.6 e143/135 9:615a5b3f:::benchmark_data_smithi001_156540_object99916:head v 143'6168, mlcod=143'6163)", 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:14.285283+0000", 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 9.906035717, 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016776527999999999, 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.273 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285283+0000", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285283+0000", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285284+0000", 2024-09-14T04:05:24.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0070000000000001e-06 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285332+0000", 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8065999999999997e-05 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285346+0000", 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4551e-05 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.285355+0000", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.884e-06 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.291305+0000", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059495559999999999 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.291362+0000", 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7540999999999997e-05 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.301865+0000", 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010502855 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:14.302059+0000", 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019406800000000001 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.277 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:104451 9.0 e143/135 9:00e83451:::benchmark_data_smithi001_156540_object104450:head v 143'6451, mlcod=143'6449)", 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:18.470087+0000", 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 5.7212316249999997, 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018601520999999999, 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470087+0000", 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.278 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470087+0000", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470088+0000", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8299999999999995e-07 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470115+0000", 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6801e-05 2024-09-14T04:05:24.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470131+0000", 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6331999999999999e-05 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470140+0000", 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.867e-06 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470451+0000", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00031161200000000001 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.470531+0000", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9994999999999996e-05 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.488647+0000", 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01811573 2024-09-14T04:05:24.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.488688+0000", 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1201000000000001e-05 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:104454 9.0 e143/135 9:09a33896:::benchmark_data_smithi001_156540_object104453:head v 143'6452, mlcod=143'6449)", 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:18.471923+0000", 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 5.719395016, 2024-09-14T04:05:24.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016794441, 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471923+0000", 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471923+0000", 2024-09-14T04:05:24.283 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471924+0000", 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1699999999999997e-07 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471948+0000", 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3533999999999999e-05 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471962+0000", 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4378e-05 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.471969+0000", 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6599999999999998e-06 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.477670+0000", 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057015260000000002 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.477719+0000", 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8664e-05 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.488692+0000", 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010973131000000001 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.488718+0000", 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5831000000000001e-05 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:107670 9.0 e143/135 9:06fa2f2c:::benchmark_data_smithi001_156540_object107669:head v 143'6632, mlcod=143'6630)", 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:21.709248+0000", 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.4820702749999999, 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016116991000000001, 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.287 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709248+0000", 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709248+0000", 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709249+0000", 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2720000000000001e-06 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709279+0000", 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9229000000000001e-05 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709299+0000", 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0146e-05 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709309+0000", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9650000000000003e-06 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709497+0000", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018804800000000001 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.709564+0000", 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6915000000000006e-05 2024-09-14T04:05:24.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.725269+0000", 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015705646 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.725365+0000", 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.577e-05 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.291 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:107671 9.0 e143/135 9:0ad4563c:::benchmark_data_smithi001_156540_object107670:head v 143'6633, mlcod=143'6630)", 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:21.710615+0000", 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.4807030989999999, 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019884421999999999, 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710615+0000", 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710615+0000", 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710616+0000", 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9699999999999995e-07 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710663+0000", 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7293999999999998e-05 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710678+0000", 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4918e-05 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.710686+0000", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1170000000000004e-06 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.714875+0000", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041880859999999997 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.714930+0000", 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5322999999999999e-05 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.730273+0000", 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01534311 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.730500+0000", 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000226777 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.296 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:107673 9.2 e143/135 9:4590fa59:::benchmark_data_smithi001_156540_object107672:head v 143'6688, mlcod=143'6685)", 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:21.712588+0000", 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.4787302589999998, 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018167863999999999, 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712588+0000", 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.297 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712588+0000", 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712589+0000", 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1769999999999999e-06 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712625+0000", 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5197999999999997e-05 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712643+0000", 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8215000000000001e-05 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.712652+0000", 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9490000000000002e-06 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.713871+0000", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012196069999999999 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.713892+0000", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0591e-05 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.730566+0000", 2024-09-14T04:05:24.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016673914000000001 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:21.730756+0000", 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000190213 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:108718 9.2 e143/135 9:4f1ee136:::benchmark_data_smithi001_156540_object108717:head v 143'6747, mlcod=143'6744)", 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:22.725984+0000", 2024-09-14T04:05:24.301 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 1.465334369, 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017684199000000001, 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.725984+0000", 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.302 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.725984+0000", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.725985+0000", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1650000000000001e-06 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.728008+0000", 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020225849999999999 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.728023+0000", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5322999999999999e-05 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.728031+0000", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.413e-06 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.728217+0000", 2024-09-14T04:05:24.304 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018662 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.728334+0000", 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116898 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.743459+0000", 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015125431 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.305 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:22.743668+0000", 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020876400000000001 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:109235 9.0 e143/135 9:0bf27294:::benchmark_data_smithi001_156540_object109234:head v 143'6742, mlcod=143'6739)", 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:23.174914+0000", 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 1.0164040910000001, 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016213200000000001, 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.306 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174914+0000", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174914+0000", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174915+0000", 2024-09-14T04:05:24.307 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5499999999999997e-07 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174938+0000", 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2877000000000001e-05 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174954+0000", 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5781999999999999e-05 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.308 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.309 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.174961+0000", 2024-09-14T04:05:24.309 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8000000000000001e-06 2024-09-14T04:05:24.309 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.309 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.176388+0000", 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014272340000000001 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.176408+0000", 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0044999999999999e-05 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.191034+0000", 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014626424000000001 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.191128+0000", 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3183000000000006e-05 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:109724 9.2 e143/135 9:4104f73f:::benchmark_data_smithi001_156540_object109723:head v 143'6810, mlcod=143'6807)", 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:23.641863+0000", 2024-09-14T04:05:24.311 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.54945581300000002, 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016016301, 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641863+0000", 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641863+0000", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641864+0000", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0330000000000001e-06 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641887+0000", 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2929999999999999e-05 2024-09-14T04:05:24.313 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641904+0000", 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7487999999999999e-05 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.641912+0000", 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1140000000000006e-06 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.314 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.645106+0000", 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031938719999999999 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.645154+0000", 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7992999999999999e-05 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.657812+0000", 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012657765 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.657879+0000", 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7106000000000004e-05 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:109735 9.0 e143/135 9:0b31b701:::benchmark_data_smithi001_156540_object109734:head v 143'6772, mlcod=143'6770)", 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:23.655157+0000", 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.53616096300000005, 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016073943, 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.316 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.655157+0000", 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.655157+0000", 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.655159+0000", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1400000000000001e-06 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.661976+0000", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0068170169999999999 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.661996+0000", 2024-09-14T04:05:24.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0030999999999998e-05 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.662002+0000", 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7519999999999999e-06 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.662062+0000", 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9438000000000002e-05 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.662089+0000", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6767999999999999e-05 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.671135+0000", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0090465470000000003 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.671231+0000", 2024-09-14T04:05:24.320 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6249999999999995e-05 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:109843 9.0 e143/135 9:06eb5f97:::benchmark_data_smithi001_156540_object109842:head v 143'6777, mlcod=143'6774)", 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:23.773433+0000", 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.41788521299999998, 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022741059000000001, 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.321 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773433+0000", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773433+0000", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773434+0000", 2024-09-14T04:05:24.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2900000000000002e-07 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773539+0000", 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000104965 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773553+0000", 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4385e-05 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.773560+0000", 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9210000000000003e-06 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.777739+0000", 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004178635 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.777805+0000", 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6471999999999996e-05 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.796077+0000", 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018271224999999999 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.796174+0000", 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7527000000000003e-05 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:109845 9.0 e143/135 9:0c1fa7b2:::benchmark_data_smithi001_156540_object109844:head v 143'6778, mlcod=143'6774)", 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:23.780114+0000", 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.411204136, 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016126556, 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780114+0000", 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780114+0000", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780115+0000", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2399999999999997e-07 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780140+0000", 2024-09-14T04:05:24.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4488999999999999e-05 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780155+0000", 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5809000000000001e-05 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.780162+0000", 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0230000000000004e-06 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.328 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.781383+0000", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012205129999999999 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.781430+0000", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6740999999999999e-05 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.796178+0000", 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014747911000000001 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:24.329 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:23.796241+0000", 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3245999999999997e-05 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:24.330 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:05:24.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:24.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:24.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:24.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:24.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:24.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:24.718 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:24.756 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:24.799 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 110941 110925 100.448 66.625 0.00670899 0.00995049 2024-09-14T04:05:24.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:24.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:24.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:24.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:25.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:25.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:25.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:25.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:25.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:25.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:25.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:25 smithi001 ceph-mon[107399]: pgmap v498: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-14T04:05:25.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:25.450 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:25.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:25.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:25.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:25.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:25 smithi001 ceph-mon[112165]: pgmap v498: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-14T04:05:25.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:25.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 112018 112002 99.9726 67.3125 0.00747046 0.00999788 2024-09-14T04:05:25.830 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:25 smithi012 ceph-mon[101180]: pgmap v498: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-14T04:05:25.866 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:25.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:25.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:26.008 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:26.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:05:26.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:26.206 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:26.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:26.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:26.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:26.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:26.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:26.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:26.655 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:26.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:26.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:26.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 113053 113037 99.4756 64.6875 0.00792807 0.0100462 2024-09-14T04:05:26.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:26.928 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:26.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:26] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-09-14T04:05:27.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:27.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:27.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:27.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:27.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:27.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:27.430 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:27 smithi001 ceph-mon[107399]: pgmap v499: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:27.430 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:27 smithi001 ceph-mon[112165]: pgmap v499: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:27.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:27.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-14T04:05:27.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:27.586 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:27.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:27.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:27.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 114064 114048 98.9716 63.1875 0.0279114 0.0100973 2024-09-14T04:05:27.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:27 smithi012 ceph-mon[101180]: pgmap v499: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:05:27.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:27.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:27.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:28.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:28.062 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:28.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:28.210 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:28.248 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:05:28.248 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:05:28.248 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:05:28.249 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:05:28.250 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:05:28.250 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:05:28.250 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:05:28.250 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:05:28.250 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:05:28.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:28.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:28.380 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:05:28.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:28.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:28.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:28.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2386693938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:05:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2386693938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:05:28.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:28.746 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:28.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:28.801 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 115002 114986 98.4185 58.625 0.0287105 0.0101562 2024-09-14T04:05:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2386693938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:05:28.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:28.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:28.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:28.998 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:29.036 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:29.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:29.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:29.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:29.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:29.312 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:29.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:29.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:05:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[107399]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T04:05:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[107399]: pgmap v501: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:05:29.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:29.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:05:29.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[112165]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T04:05:29.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:29 smithi001 ceph-mon[112165]: pgmap v501: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:05:29.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:29.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:29.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:29.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:29.801 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 116063 116047 97.9847 66.3125 0.015082 0.0102006 2024-09-14T04:05:29.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:29 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:05:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/406036851' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:05:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:29 smithi012 ceph-mon[101180]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T04:05:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:29 smithi012 ceph-mon[101180]: pgmap v501: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:05:29.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:29.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:29.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:29.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:30.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:30.013 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:30.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:30.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:30.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:30.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:30.530 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.565 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:30.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:30.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.707 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:30.801 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 117106 117090 97.5474 65.1875 0.0277713 0.0102456 2024-09-14T04:05:30.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:30.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:30.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:30.990 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:30.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:31.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:31.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:31.229 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:31.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:05:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.265 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:31.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:31.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:31.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:31.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:31.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:31.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:31.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:31.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:05:31.717 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:31 smithi001 ceph-mon[107399]: pgmap v502: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-14T04:05:31.717 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:31 smithi001 ceph-mon[112165]: pgmap v502: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-14T04:05:31.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:31.801 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 118079 118063 97.064 60.8125 0.0138992 0.0102969 2024-09-14T04:05:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:31.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:31 smithi012 ceph-mon[101180]: pgmap v502: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-14T04:05:31.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:31.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:31.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:32.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:32.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:32.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:32.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:32.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:32.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:32.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:32.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:32.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:32.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:32.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:32.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:32.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:32.801 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 119065 119049 96.6038 61.625 0.0202535 0.010346 2024-09-14T04:05:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.818 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:32.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:32.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:32.967 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:32.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:32.986 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:05:32.986 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:05:33.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.107 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:33.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:33.209 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:33.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:33.347 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:33.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:33.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:33.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:33.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:33.609 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:33 smithi001 ceph-mon[107399]: pgmap v503: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:33.609 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:33 smithi001 ceph-mon[112165]: pgmap v503: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:33.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:33.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:05:33.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:33.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:33.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 120086 120070 96.1797 63.8125 0.0320894 0.0103924 2024-09-14T04:05:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:33.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:33.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:33 smithi012 ceph-mon[101180]: pgmap v503: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:33.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:34.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:34.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:34.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:34.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:34.226 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:34.227 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:34.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:34.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:34.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:34.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:34.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:34.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:34.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:34.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:34.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:34.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:34.804 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 121016 121000 95.698 58.125 0.0142149 0.0104443 2024-09-14T04:05:34.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:34.849 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:34.951 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:34.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:34.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:35.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:35.097 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:35.097 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:35.128 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:35.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:35.266 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:35.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:35.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:35.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:35.545 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:35.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:35.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:35.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:35.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:05:35.804318+0000 min lat: 0.00124478 max lat: 0.175154 avg lat: 0.0104883 2024-09-14T04:05:35.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:05:35.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 122025 122009 95.2901 63.0625 0.00760844 0.0104883 2024-09-14T04:05:35.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:35.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:35.946 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:35.959 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:35.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:36.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:36.100 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:36.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:36.240 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:36.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:35 smithi012 ceph-mon[101180]: pgmap v504: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:36.350 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:35 smithi001 ceph-mon[107399]: pgmap v504: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:36.350 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:35 smithi001 ceph-mon[112165]: pgmap v504: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:05:36.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:36.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-14T04:05:36.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:36.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:36.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:36.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:36.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:36.788 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:36.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 123003 122987 94.8663 61.125 0.0195667 0.0105364 2024-09-14T04:05:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:36.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:36.857 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:36] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-14T04:05:36.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:36.977 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:36.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:37.079 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:37.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:37.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:37.224 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:37.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:37.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:37.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:37.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:37.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:37.644 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:37 smithi001 ceph-mon[107399]: pgmap v505: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:37.644 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:37 smithi001 ceph-mon[112165]: pgmap v505: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:37.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:37.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:37.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:37.808 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 124012 123996 94.4764 63.0625 0.0255007 0.0105799 2024-09-14T04:05:37.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:37 smithi012 ceph-mon[101180]: pgmap v505: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:05:37.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:37.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:37.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:37.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:38.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:38.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:38.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:38.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:38.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:38.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:38.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:38.537 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:38.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:38.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:38.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:38.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 125002 124986 94.0816 61.875 0.012868 0.0106238 2024-09-14T04:05:38.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:38.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:38.941 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:05:38.941 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:05:38.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:38.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:39.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:39.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:39.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:39.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:39.312 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3057057257' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:39.312 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3057057257' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:39.339 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:39.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:05:39.387 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:05:39.388 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:05:39.388 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:05:39.388 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:05:39.388 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:05:39.422 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:05:39.422 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:05:39.453 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:05:39.453 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:05:39.476 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:39.490 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:05:39.491 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:05:39.522 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3057057257' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:39.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.659 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:39.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:39.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:39.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 125947 125931 93.6647 59.0625 0.00919895 0.0106717 2024-09-14T04:05:39.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:05:39.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:39.986 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:39.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:40.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:40.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:40.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:40.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:40.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:40.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:40.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:40.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:40.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:40.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:40 smithi012 ceph-mon[101180]: pgmap v506: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-14T04:05:40.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:40.697 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:40 smithi001 ceph-mon[107399]: pgmap v506: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-14T04:05:40.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:40 smithi001 ceph-mon[112165]: pgmap v506: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-14T04:05:40.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:40.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:40.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 126892 126876 93.2577 59.0625 0.0145879 0.0107182 2024-09-14T04:05:40.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:40.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:40.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:40.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:40.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:40.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:41.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:41.128 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:41.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:41.168 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:41.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:41.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:41.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:41.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:41.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:41 smithi001 ceph-mon[107399]: pgmap v507: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:05:41.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:41 smithi001 ceph-mon[112165]: pgmap v507: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:05:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:41.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:41.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:41.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:41.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:41.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:41.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 127840 127824 92.8623 59.25 0.0217309 0.0107636 2024-09-14T04:05:41.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:41.841 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:41 smithi012 ceph-mon[101180]: pgmap v507: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:05:41.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:41.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:41.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:41.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:42.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:42.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:42.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:42.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:42.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:42.406 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:05:42.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:42.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:42.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:42.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:42.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:42.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:42.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:42.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:42.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 128759 128743 92.4552 57.4375 0.0192164 0.0108111 2024-09-14T04:05:42.839 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:42.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:05:42.941 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:42.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:42.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:42.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:43.082 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:43.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:43.208 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:05:43.233 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:43.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:43.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:43.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:43.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:43.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:43.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:43.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[107399]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[107399]: pgmap v509: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[112165]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T04:05:43.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:43 smithi001 ceph-mon[112165]: pgmap v509: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:05:43.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 129713 129697 92.0821 59.625 0.0147467 0.0108551 2024-09-14T04:05:43.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:43 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:05:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:43 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:05:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:43 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4006456354' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:05:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:43 smithi012 ceph-mon[101180]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T04:05:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:43 smithi012 ceph-mon[101180]: pgmap v509: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:05:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:43.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:43.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:43.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:44.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:44.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:44.175 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:44.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:44.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:44.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:44.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:44.568 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:44.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:44.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:44.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:44.810 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 130653 130637 91.7077 58.75 0.013716 0.0108995 2024-09-14T04:05:44.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:44.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:44.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:44.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:45.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:45.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:45.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:45.270 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:45.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:45.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:45.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:45.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:45.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:45.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:45.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:45.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0072 2024-09-14T04:05:45.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 131591 2024-09-14T04:05:45.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:05:45.811 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 91.3754 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 38.6832 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 239.75 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 57.4375 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1462 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 618.931 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3836 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 919 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0109398 2024-09-14T04:05:45.812 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.00782689 2024-09-14T04:05:45.813 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 0.175154 2024-09-14T04:05:45.813 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00124478 2024-09-14T04:05:45.817 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:05:45.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:45.872 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:45.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:45.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:46.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:46.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:46.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:46.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:46.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:46.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:46.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:46.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:46.476 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:46.505 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-14T04:05:46.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.556 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:46.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:46.641 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:46 smithi001 ceph-mon[107399]: pgmap v510: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-14T04:05:46.641 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:46 smithi001 ceph-mon[112165]: pgmap v510: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-14T04:05:46.658 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:46.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:46.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:46 smithi012 ceph-mon[101180]: pgmap v510: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-09-14T04:05:46.873 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:46.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:46.903 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:46] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-14T04:05:46.975 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:47.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:47.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:47.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:47.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:47.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:47.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.432 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:47.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:47.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:47.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:47.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:47.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:47.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:47.852 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:05:47.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:47.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:47.964 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:48.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:48.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:48.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:48.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.301 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:48.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:48.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:48.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:48 smithi001 ceph-mon[112165]: pgmap v511: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:48 smithi001 ceph-mon[107399]: pgmap v511: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:48.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.576 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:48.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:48.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:48.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:48.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:48 smithi012 ceph-mon[101180]: pgmap v511: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:48.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:48.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:48.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:48.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:49.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:49.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:49.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:49.271 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:49.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:49.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:49.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:49.537 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:49.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:49.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:49.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:49.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:49.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:49.959 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:49.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:50.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:50.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:50.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:50.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:50.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:50.430 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:05:50.430 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:05:50.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:05:50.430 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60280 9.2 9:4b6aed7c:::benchmark_data_smithi001_156540_object60279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.187267+0000", 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.007225136000002, 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.100924869, 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60280 2024-09-14T04:05:50.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187267+0000", 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187267+0000", 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187268+0000", 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.437e-06 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187292+0000", 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3543000000000001e-05 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187303+0000", 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1817000000000001e-05 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187310+0000", 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7379999999999997e-06 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187366+0000", 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6371e-05 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187469+0000", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102624 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.187595+0000", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012585299999999999 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191024+0000", 2024-09-14T04:05:50.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034289469999999999 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194013+0000", 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002988966 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288078+0000", 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094065317999999995 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288139+0000", 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1135000000000003e-05 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288191+0000", 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2120000000000002e-05 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:60283 9.7 e141/135 9:e45749ca:::benchmark_data_smithi001_156540_object60282:head v 141'3719, mlcod=141'3716)", 2024-09-14T04:05:50.437 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.189661+0000", 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.004830373000004, 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.098708053000000004, 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189661+0000", 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189661+0000", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189662+0000", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.06e-06 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189720+0000", 2024-09-14T04:05:50.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7846000000000001e-05 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189730+0000", 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0156000000000001e-05 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189737+0000", 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4370000000000004e-06 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189837+0000", 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9751000000000001e-05 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.189854+0000", 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7088e-05 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288249+0000", 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.098395675000000002 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288369+0000", 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012004 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60287 9.2 9:4882f227:::benchmark_data_smithi001_156540_object60286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.190810+0000", 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.003681227000001, 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10482518, 2024-09-14T04:05:50.442 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60287 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190810+0000", 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190810+0000", 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190812+0000", 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5009999999999999e-06 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190841+0000", 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8912e-05 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190852+0000", 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1408e-05 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190860+0000", 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3640000000000001e-06 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191069+0000", 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020940800000000001 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191571+0000", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00050185900000000001 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191705+0000", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013370400000000001 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194021+0000", 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023159249999999999 2024-09-14T04:05:50.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295466+0000", 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.101445383 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295526+0000", 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0276999999999999e-05 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295586+0000", 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9901000000000001e-05 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295636+0000", 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9537999999999999e-05 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.449 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:60286 9.3 e141/135 9:ccc379e2:::benchmark_data_smithi001_156540_object60285:head v 141'3812, mlcod=141'3809)", 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.190963+0000", 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.003528183, 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.097117835, 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190963+0000", 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190963+0000", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.190964+0000", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4899999999999995e-07 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191013+0000", 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9178000000000002e-05 2024-09-14T04:05:50.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191022+0000", 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4460000000000001e-06 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191028+0000", 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5030000000000002e-06 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194425+0000", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003397172 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194444+0000", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8425000000000001e-05 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197139+0000", 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026954470000000001 2024-09-14T04:05:50.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288081+0000", 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.090942115000000004 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60289 9.1 9:8c40eb0d:::benchmark_data_smithi001_156540_object60288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.191126+0000", 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.003366060000005, 2024-09-14T04:05:50.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.099786224000000007, 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60289 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191126+0000", 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191126+0000", 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191126+0000", 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2200000000000003e-07 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191149+0000", 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2367000000000001e-05 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191158+0000", 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4749999999999999e-06 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191165+0000", 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3570000000000003e-06 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193903+0000", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027385700000000001 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194031+0000", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012757799999999999 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196362+0000", 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023308859999999999 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288089+0000", 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.091727848000000001 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.290588+0000", 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002498456 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.290727+0000", 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013923599999999999 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.290783+0000", 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5509e-05 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.290912+0000", 2024-09-14T04:05:50.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012922000000000001 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60291 9.8 9:15cbf803:::benchmark_data_smithi001_156540_object60290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.191598+0000", 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.002893334000007, 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.097058640000000002, 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.461 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60291 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191598+0000", 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191598+0000", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191599+0000", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4399999999999999e-07 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191660+0000", 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0955000000000002e-05 2024-09-14T04:05:50.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191669+0000", 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4980000000000006e-06 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191675+0000", 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0310000000000004e-06 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191729+0000", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4617999999999998e-05 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.191909+0000", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018001000000000001 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.192068+0000", 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015875700000000001 2024-09-14T04:05:50.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194485+0000", 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024166679999999999 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288516+0000", 2024-09-14T04:05:50.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094031396000000003 2024-09-14T04:05:50.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288564+0000", 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7691000000000001e-05 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288613+0000", 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9508000000000001e-05 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288657+0000", 2024-09-14T04:05:50.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3764000000000003e-05 2024-09-14T04:05:50.470 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.470 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.470 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60293 9.8 9:1dc8b0f7:::benchmark_data_smithi001_156540_object60292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.193976+0000", 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.000516028999996, 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.095312728999999999, 2024-09-14T04:05:50.471 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.472 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60293 2024-09-14T04:05:50.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.473 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193976+0000", 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193976+0000", 2024-09-14T04:05:50.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193977+0000", 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5889999999999999e-06 2024-09-14T04:05:50.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194004+0000", 2024-09-14T04:05:50.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7081e-05 2024-09-14T04:05:50.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194015+0000", 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1161e-05 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194022+0000", 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9779999999999999e-06 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194511+0000", 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048878699999999997 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194698+0000", 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000186847 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194903+0000", 2024-09-14T04:05:50.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020506800000000001 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288708+0000", 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.093805056999999997 2024-09-14T04:05:50.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288744+0000", 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5655999999999999e-05 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.289195+0000", 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000451209 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.289252+0000", 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6611999999999999e-05 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.289288+0000", 2024-09-14T04:05:50.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6684000000000003e-05 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:60288 9.f e141/135 9:f0df53d1:::benchmark_data_smithi001_156540_object60287:head v 141'3743, mlcod=141'3741)", 2024-09-14T04:05:50.485 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.193983+0000", 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.000508280999995, 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094418104000000003, 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.486 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193983+0000", 2024-09-14T04:05:50.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193983+0000", 2024-09-14T04:05:50.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.193984+0000", 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7899999999999997e-07 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194022+0000", 2024-09-14T04:05:50.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8105999999999997e-05 2024-09-14T04:05:50.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194034+0000", 2024-09-14T04:05:50.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1358999999999999e-05 2024-09-14T04:05:50.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194040+0000", 2024-09-14T04:05:50.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5950000000000004e-06 2024-09-14T04:05:50.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194911+0000", 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00087114199999999999 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194928+0000", 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6629e-05 2024-09-14T04:05:50.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288372+0000", 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.093444365000000001 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288401+0000", 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9028999999999999e-05 2024-09-14T04:05:50.494 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.495 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.495 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.496 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:60292 9.7 e141/135 9:e29eaa91:::benchmark_data_smithi001_156540_object60291:head v 141'3720, mlcod=141'3716)", 2024-09-14T04:05:50.496 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.194053+0000", 2024-09-14T04:05:50.496 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 66.000438313000004, 2024-09-14T04:05:50.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094375094000000007, 2024-09-14T04:05:50.497 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.497 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.497 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.497 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194053+0000", 2024-09-14T04:05:50.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194053+0000", 2024-09-14T04:05:50.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194054+0000", 2024-09-14T04:05:50.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2599999999999998e-07 2024-09-14T04:05:50.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.501 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.501 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194085+0000", 2024-09-14T04:05:50.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0854e-05 2024-09-14T04:05:50.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194093+0000", 2024-09-14T04:05:50.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1650000000000006e-06 2024-09-14T04:05:50.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.194098+0000", 2024-09-14T04:05:50.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4879999999999998e-06 2024-09-14T04:05:50.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.195643+0000", 2024-09-14T04:05:50.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001544428 2024-09-14T04:05:50.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.195661+0000", 2024-09-14T04:05:50.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8213e-05 2024-09-14T04:05:50.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288404+0000", 2024-09-14T04:05:50.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.092742878000000001 2024-09-14T04:05:50.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288428+0000", 2024-09-14T04:05:50.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4641999999999999e-05 2024-09-14T04:05:50.508 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.508 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.508 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.509 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60296 9.1 9:851f7ccd:::benchmark_data_smithi001_156540_object60295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.509 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.196195+0000", 2024-09-14T04:05:50.509 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.998296773999996, 2024-09-14T04:05:50.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10414416899999999, 2024-09-14T04:05:50.510 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.510 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.510 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.510 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.511 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.511 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.511 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60296 2024-09-14T04:05:50.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.511 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196195+0000", 2024-09-14T04:05:50.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196195+0000", 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.514 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.514 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196196+0000", 2024-09-14T04:05:50.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.266e-06 2024-09-14T04:05:50.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.515 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196219+0000", 2024-09-14T04:05:50.515 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2636000000000001e-05 2024-09-14T04:05:50.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196239+0000", 2024-09-14T04:05:50.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0423000000000001e-05 2024-09-14T04:05:50.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196247+0000", 2024-09-14T04:05:50.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6879999999999999e-06 2024-09-14T04:05:50.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288207+0000", 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.091959618000000007 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288348+0000", 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000141657 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:05:50.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288472+0000", 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123666 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.290954+0000", 2024-09-14T04:05:50.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024817709999999998 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.291991+0000", 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0010374620000000001 2024-09-14T04:05:50.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.300254+0000", 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0082631509999999998 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.300301+0000", 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7012000000000003e-05 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.300339+0000", 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7818999999999998e-05 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60299 9.c 9:362a9ff1:::benchmark_data_smithi001_156540_object60298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.196264+0000", 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.998227498999995, 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.099588029999999994, 2024-09-14T04:05:50.525 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.526 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60299 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196264+0000", 2024-09-14T04:05:50.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196264+0000", 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196265+0000", 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1500000000000004e-07 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196289+0000", 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4060999999999999e-05 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196298+0000", 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1239999999999994e-06 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196304+0000", 2024-09-14T04:05:50.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1789999999999996e-06 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196365+0000", 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1173000000000005e-05 2024-09-14T04:05:50.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196564+0000", 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000199125 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196731+0000", 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016662799999999999 2024-09-14T04:05:50.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.291881+0000", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.095149732000000001 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.292877+0000", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00099608599999999998 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295750+0000", 2024-09-14T04:05:50.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028733690000000002 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295807+0000", 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6474000000000002e-05 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295852+0000", 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5463999999999997e-05 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.44714.0:60280 9.2 e141/135 ondisk, result = 0)", 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.196442+0000", 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.998049473999998, 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.091758930000000002, 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196442+0000", 2024-09-14T04:05:50.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196442+0000", 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196443+0000", 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7199999999999998e-07 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196448+0000", 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0649999999999998e-06 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196456+0000", 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2330000000000006e-06 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.196462+0000", 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8170000000000004e-06 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288061+0000", 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.091598924999999998 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288071+0000", 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8279999999999998e-06 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288201+0000", 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013039 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.542 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.44714.0:60287 9.2 e141/135 ondisk, result = 0)", 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.197159+0000", 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.997332314999994, 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.098316135999999998, 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197159+0000", 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197159+0000", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197160+0000", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0399999999999996e-07 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197164+0000", 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1370000000000004e-06 2024-09-14T04:05:50.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288110+0000", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.090945680000000001 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288118+0000", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9670000000000001e-06 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295453+0000", 2024-09-14T04:05:50.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073350150000000003 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295460+0000", 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6210000000000004e-06 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295475+0000", 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5112e-05 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.44714.0:60287 9.2 e141/135 ondisk, result = 0)", 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.197189+0000", 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.997303091000006, 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.098458390000000007, 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197189+0000", 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197189+0000", 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197189+0000", 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2499999999999995e-07 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.197192+0000", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9799999999999998e-06 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288312+0000", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.091120266000000005 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.288319+0000", 2024-09-14T04:05:50.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6460000000000004e-06 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295515+0000", 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0071957289999999997 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295521+0000", 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4629999999999998e-06 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295647+0000", 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012578099999999999 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:60301 9.c 9:335d9db5:::benchmark_data_smithi001_156540_object60300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:04:43.199330+0000", 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 65.995161272999994, 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10047096699999999, 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.551 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 60301 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.199330+0000", 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.199330+0000", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.199332+0000", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2100000000000001e-06 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.199351+0000", 2024-09-14T04:05:50.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9837e-05 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.287860+0000", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.088509031000000002 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.287902+0000", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1149000000000002e-05 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.292906+0000", 2024-09-14T04:05:50.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0050046980000000001 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.293016+0000", 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000110124 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.293146+0000", 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000129729 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.295859+0000", 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027131059999999999 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.299588+0000", 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037290660000000001 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.299689+0000", 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101037 2024-09-14T04:05:50.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.299748+0000", 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9018000000000003e-05 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:04:43.299801+0000", 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2961999999999999e-05 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:103170 9.2 9:4092b69e:::benchmark_data_smithi001_156540_object103169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:17.248562+0000", 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 31.945929218, 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094285748000000003, 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 103170 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.558 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.248562+0000", 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.248562+0000", 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.248563+0000", 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0400000000000005e-07 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.248805+0000", 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024196299999999999 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.310160+0000", 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.061354909999999999 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.310187+0000", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6811000000000001e-05 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.314742+0000", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045551599999999999 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.314889+0000", 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014733300000000001 2024-09-14T04:05:50.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.315028+0000", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000138992 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.328533+0000", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01350471 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.328547+0000", 2024-09-14T04:05:50.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3754000000000001e-05 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.342756+0000", 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014209019999999999 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.342807+0000", 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1390000000000001e-05 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:17.342848+0000", 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0901e-05 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:104756 9.2 9:4bbb9dbc:::benchmark_data_smithi001_156540_object104755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:18.736887+0000", 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 30.457604657000001, 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.086225045, 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.564 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 104756 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736887+0000", 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736887+0000", 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736888+0000", 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0100000000000004e-07 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736910+0000", 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2014999999999998e-05 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736920+0000", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0429e-05 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736928+0000", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3830000000000003e-06 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.736983+0000", 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5055000000000001e-05 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.737088+0000", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105002 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.737217+0000", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012942699999999999 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.813272+0000", 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.076054679 2024-09-14T04:05:50.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.821625+0000", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.008353447 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.823014+0000", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0013885 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.823067+0000", 2024-09-14T04:05:50.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3514999999999998e-05 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.823112+0000", 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4792000000000001e-05 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:104762 9.2 9:4e30b4c7:::benchmark_data_smithi001_156540_object104761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:18.740888+0000", 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 30.453603774000001, 2024-09-14T04:05:50.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.090627994000000003, 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 104762 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740888+0000", 2024-09-14T04:05:50.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740888+0000", 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740889+0000", 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1999999999999999e-07 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740913+0000", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4158000000000001e-05 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740921+0000", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6410000000000008e-06 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.740928+0000", 2024-09-14T04:05:50.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2790000000000004e-06 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.741008+0000", 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0801000000000001e-05 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.741120+0000", 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011127 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:05:50.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.741261+0000", 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000141163 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.813306+0000", 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.072044695000000006 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.821633+0000", 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0083270619999999997 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.831415+0000", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097827609999999992 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.831470+0000", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4721e-05 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:18.831516+0000", 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5723000000000003e-05 2024-09-14T04:05:50.576 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:134635 9.1 9:8993508d:::benchmark_data_smithi001_156540_object3041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:48.936878+0000", 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.25761347699999998, 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.089578636000000003, 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.577 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 134635 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936878+0000", 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936878+0000", 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936879+0000", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.066e-06 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936884+0000", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.403e-06 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936895+0000", 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1165999999999999e-05 2024-09-14T04:05:50.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.936902+0000", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.029e-06 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.937020+0000", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117953 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.937126+0000", 2024-09-14T04:05:50.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106257 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.937241+0000", 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115272 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.948767+0000", 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011525695000000001 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.948809+0000", 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1845999999999999e-05 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026316+0000", 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.077507254999999997 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026411+0000", 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4981000000000004e-05 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026457+0000", 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5713000000000001e-05 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:134641 9.1 9:82622573:::benchmark_data_smithi001_156540_object3047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:48.939529+0000", 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.25496278700000002, 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.087090844000000001, 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:05:50.583 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 134641 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939529+0000", 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939529+0000", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939530+0000", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8899999999999998e-07 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939533+0000", 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.411e-06 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939543+0000", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9080000000000007e-06 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.939550+0000", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8009999999999998e-06 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.942044+0000", 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024936870000000001 2024-09-14T04:05:50.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.942149+0000", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010574299999999999 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.942253+0000", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103658 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:48.948872+0000", 2024-09-14T04:05:50.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0066185610000000002 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.020123+0000", 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.071251936000000002 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026509+0000", 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063851300000000001 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026585+0000", 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6420000000000004e-05 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.026620+0000", 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4700000000000003e-05 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:05:50.589 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:05:50.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:50.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:50.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:50.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:50.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:50.905 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:50.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:51.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:51.049 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:51.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:51.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[112165]: pgmap v512: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[107399]: pgmap v512: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:51.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:51.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:50 smithi012 ceph-mon[101180]: pgmap v512: 89 pgs: 89 active+clean; 8.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 961 op/s 2024-09-14T04:05:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:05:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:51.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.464 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:51.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:51.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:51.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:51.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:51.747 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:51.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:51.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:51.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:51.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:52.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:52.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:52.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:52.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:52.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:52.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:52.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:52.571 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:52.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:52.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:52.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:52.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:52.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:52.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:52.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:53.098 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:53.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:52 smithi001 ceph-mon[112165]: pgmap v513: 89 pgs: 89 active+clean; 7.9 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 947 op/s 2024-09-14T04:05:53.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:52 smithi001 ceph-mon[107399]: pgmap v513: 89 pgs: 89 active+clean; 7.9 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 947 op/s 2024-09-14T04:05:53.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:53.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:53.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:52 smithi012 ceph-mon[101180]: pgmap v513: 89 pgs: 89 active+clean; 7.9 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 947 op/s 2024-09-14T04:05:53.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:53.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:53.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:53.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:53.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:53.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:53.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:53.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:53.834 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:53.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:53.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:53.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:54.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:54.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:54.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:54.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:54.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:54.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:54.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:54.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:54.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:54.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:54.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:54.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:54.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:54.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:54.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:05:54.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:54.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:54.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:55.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:55.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:55.208 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:54 smithi001 ceph-mon[107399]: pgmap v514: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 93 B/s rd, 27 MiB/s wr, 861 op/s 2024-09-14T04:05:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:54 smithi001 ceph-mon[112165]: pgmap v514: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 93 B/s rd, 27 MiB/s wr, 861 op/s 2024-09-14T04:05:55.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:55.227 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:55.231 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:05:55.231 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:05:55.271 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:05:55.272 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:05:55.281 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:05:55.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:55.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:05:55.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:05:55.342 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:05:55.343 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:05:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:54 smithi012 ceph-mon[101180]: pgmap v514: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 93 B/s rd, 27 MiB/s wr, 861 op/s 2024-09-14T04:05:55.377 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:05:55.378 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:05:55.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:55.414 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:05:55.414 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:05:55.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:55.449 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:05:55.450 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:05:55.480 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:05:55.481 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:05:55.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:55.568 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:55.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:55.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:55.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:55.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:55.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:55.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:55.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:55.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:55.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:56.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1595353499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:56.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:56.174 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1595353499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:56.175 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1595353499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:05:56.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:56.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:56.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:05:56.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:05:56.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:56.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:56.656 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:56.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:56.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:56.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.832 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:56.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:56.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:56.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:56.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:56.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:56 smithi001 ceph-mon[112165]: pgmap v515: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 827 op/s 2024-09-14T04:05:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:56 smithi001 ceph-mon[107399]: pgmap v515: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 827 op/s 2024-09-14T04:05:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:05:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:05:56] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-09-14T04:05:57.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:57.107 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:57.209 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:57.253 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:57.271 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:56 smithi012 ceph-mon[101180]: pgmap v515: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 827 op/s 2024-09-14T04:05:57.340 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:05:57.340 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:05:57.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:57.399 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:57.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:57.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:57.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:57.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:57.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:57.806 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:57.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:05:57.951 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:57.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:05:58.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:58.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:58.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:58.211 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:05:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:58.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:58.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:58.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:58.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:58.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:58.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.513 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:58.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:58.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:58.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:58.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:58.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:05:58.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:05:58.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:05:58.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:58.952 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:58 smithi012 ceph-mon[101180]: pgmap v516: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.12k op/s 2024-09-14T04:05:58.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:58.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:59.039 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:58 smithi001 ceph-mon[112165]: pgmap v516: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.12k op/s 2024-09-14T04:05:59.039 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:58 smithi001 ceph-mon[107399]: pgmap v516: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.12k op/s 2024-09-14T04:05:59.043 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:05:59.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:59.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:59.194 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:05:59.238 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:59.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:05:59.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:59.384 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:59.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:05:59.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:05:59.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:05:59.655 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:05:59.685 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:59.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:05:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:05:59.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:05:59.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:05:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:05:59.953 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:05:59.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:05:59.970 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:06:00.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:00.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:00.169 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[107399]: pgmap v517: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 770 op/s 2024-09-14T04:06:00.170 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3786950222' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.170 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[112165]: pgmap v517: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 770 op/s 2024-09-14T04:06:00.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3786950222' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:05:59 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:00.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.233 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:00.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:00.338 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:59 smithi012 ceph-mon[101180]: pgmap v517: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.2 MiB/s wr, 770 op/s 2024-09-14T04:06:00.338 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3786950222' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.338 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:05:59 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:00.386 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:00.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:00.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:00.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:00.664 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:00.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:00.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:00.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:00.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:00.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:00.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:01.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.095 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:01.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:01.098 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:01.098 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:01.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[107399]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:01.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:00 smithi001 ceph-mon[112165]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T04:06:01.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:01.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:01.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:00 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:01.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:00 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:01.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:00 smithi012 ceph-mon[101180]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T04:06:01.379 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:01.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:01.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:01.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:01.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:01.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:01.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:01.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:01.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:06:01.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:01.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:01.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:02.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:02.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:02.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:02.216 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:02.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:02.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:02.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:02.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:02.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:02.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:02.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:02.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:02.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:02.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:02.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:02.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:03.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:02 smithi001 ceph-mon[107399]: pgmap v519: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:06:03.053 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:02 smithi001 ceph-mon[112165]: pgmap v519: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:06:03.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:03.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:03.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:03.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:03.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:02 smithi012 ceph-mon[101180]: pgmap v519: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:06:03.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:03.409 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:06:03.409 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:06:03.435 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:03.435 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:03.472 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:03.476 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:03.476 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:03.512 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:03.517 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:03.517 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:03.548 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:03.548 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:03.579 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:03.580 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:03.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:03.615 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:03.615 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:03.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:03.651 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:03.651 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:03.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:03.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:03.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:03.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:03.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:04.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:04.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:04.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.169 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:04.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:04.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:04.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3781907866' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:04.379 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3781907866' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:04.379 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3781907866' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:04.407 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.435 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:04.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:04.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:04.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:04.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:04.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:04.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:04.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:04.830 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:06:04.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:04.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:04.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:04.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:05.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:05.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:05.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:05.139 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:05.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[112165]: pgmap v520: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:06:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:05.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:05.224 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[107399]: pgmap v520: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:06:05.224 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:05.224 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:05.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:04 smithi012 ceph-mon[101180]: pgmap v520: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:06:05.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:05.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:05.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:05.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:05.397 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:05.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:05.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:05.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:05.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:05.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:05.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:05.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:06.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:06.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:06.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:06.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:06.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:06.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:06.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:06:06.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:06.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:06.554 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:06.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:06:06.555 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:06.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd out 3 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44135.0:228490 7.1 7:84104608:::benchmark_data_smithi001_117782_object102765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e91)", 2024-09-14T04:06:06.562 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:56:22.711377+0000", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.84574456400003, 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.133863858, 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44135", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2509422189", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 228490 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711377+0000", 2024-09-14T04:06:06.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711377+0000", 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711376+0000", 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711382+0000", 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2099999999999998e-06 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711394+0000", 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2276000000000001e-05 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711401+0000", 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3600000000000001e-06 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711445+0000", 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4446000000000001e-05 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711525+0000", 2024-09-14T04:06:06.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9529000000000004e-05 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.711593+0000", 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8773e-05 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.714857+0000", 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032638469999999998 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.715041+0000", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000183678 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.845143+0000", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.130101833 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.845197+0000", 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4262999999999999e-05 2024-09-14T04:06:06.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:22.845241+0000", 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4063000000000001e-05 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44135.0:246950 7.4 7:21c3652f:::benchmark_data_smithi001_117782_object121225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e92)", 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:56:47.230697+0000", 2024-09-14T04:06:06.569 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 559.32642431299996, 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.073120847, 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44135", 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2509422189", 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 246950 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230697+0000", 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230697+0000", 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230696+0000", 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230701+0000", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9490000000000004e-06 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230713+0000", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1462e-05 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230718+0000", 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:06:06.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230763+0000", 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5172000000000001e-05 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:47.230772+0000", 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5790000000000004e-06 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.273151+0000", 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0423790610000001 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.277512+0000", 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043614480000000004 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.278133+0000", 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062081599999999995 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.300236+0000", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022102522999999999 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.303688+0000", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034523169999999999 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.303738+0000", 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0516000000000002e-05 2024-09-14T04:06:06.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:56:48.303818+0000", 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9864000000000006e-05 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:67863 8.2 8:47e007e4:::benchmark_data_smithi001_125658_object67862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:57:39.349662+0000", 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 507.20745981599998, 2024-09-14T04:06:06.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.108863216, 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 67863 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349662+0000", 2024-09-14T04:06:06.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349662+0000", 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349660+0000", 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349708+0000", 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7630000000000003e-05 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349719+0000", 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.081e-05 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349724+0000", 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8139999999999997e-06 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349774+0000", 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9376000000000003e-05 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.349909+0000", 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013491600000000001 2024-09-14T04:06:06.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.350033+0000", 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012393500000000001 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.354339+0000", 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043060770000000002 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.355620+0000", 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012808089999999999 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458439+0000", 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102819385 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458483+0000", 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4132999999999999e-05 2024-09-14T04:06:06.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458525+0000", 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2126000000000003e-05 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:67864 8.2 8:4a1884a6:::benchmark_data_smithi001_125658_object67863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:57:39.351551+0000", 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 507.20557033300003, 2024-09-14T04:06:06.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107106032, 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 67864 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351551+0000", 2024-09-14T04:06:06.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351551+0000", 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351550+0000", 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351586+0000", 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6312000000000001e-05 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351603+0000", 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6694e-05 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.351611+0000", 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8229999999999995e-06 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.352208+0000", 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000597593 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.352344+0000", 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135508 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.352487+0000", 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014352500000000001 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.355628+0000", 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003140346 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356215+0000", 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058721100000000003 2024-09-14T04:06:06.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458568+0000", 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102352919 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458623+0000", 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5127000000000003e-05 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458657+0000", 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4610000000000002e-05 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:67875 8.2 8:4d888a67:::benchmark_data_smithi001_125658_object67874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:57:39.356742+0000", 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 507.20038017600001, 2024-09-14T04:06:06.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102116792, 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 67875 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356742+0000", 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356742+0000", 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:06.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356740+0000", 2024-09-14T04:06:06.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:06:06.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356772+0000", 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2634e-05 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356786+0000", 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3776e-05 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356793+0000", 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0339999999999999e-06 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356837+0000", 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4172000000000003e-05 2024-09-14T04:06:06.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.356923+0000", 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5358999999999994e-05 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.357013+0000", 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9708000000000002e-05 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.361150+0000", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041371109999999997 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.361516+0000", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036631199999999999 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458725+0000", 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.097209151999999993 2024-09-14T04:06:06.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458817+0000", 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1823000000000005e-05 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458858+0000", 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1471000000000003e-05 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:67876 8.2 8:49838f41:::benchmark_data_smithi001_125658_object67875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:57:39.358401+0000", 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 507.19872076600001, 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100592916, 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 67876 2024-09-14T04:06:06.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358401+0000", 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358401+0000", 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358400+0000", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358433+0000", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2626999999999998e-05 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358445+0000", 2024-09-14T04:06:06.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2476999999999999e-05 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358450+0000", 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3600000000000004e-06 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.358880+0000", 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042911700000000002 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.359022+0000", 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014196099999999999 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.359163+0000", 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141512 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.361288+0000", 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002124833 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.361623+0000", 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00033549699999999998 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458911+0000", 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.097287376999999994 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458961+0000", 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9761e-05 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.458994+0000", 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3413e-05 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:68240 8.2 8:41573fe1:::benchmark_data_smithi001_125658_object68239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-14T04:06:06.605 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T03:57:39.659789+0000", 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 506.89733234099998, 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10380083900000001, 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 68240 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659789+0000", 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659789+0000", 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659788+0000", 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659816+0000", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7727000000000001e-05 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659824+0000", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4649999999999994e-06 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.659830+0000", 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4979999999999997e-06 2024-09-14T04:06:06.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.660060+0000", 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000230151 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.660446+0000", 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038609899999999999 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.660905+0000", 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045849299999999998 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.663589+0000", 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026841059999999999 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.664585+0000", 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099624299999999995 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.763500+0000", 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098915055000000002 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.763552+0000", 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1691999999999999e-05 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T03:57:39.763590+0000", 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8458999999999999e-05 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:194912 8.2 8:4afbab09:::benchmark_data_smithi001_125658_object69673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.106604+0000", 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.45051789199999, 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.103295292, 2024-09-14T04:06:06.612 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 194912 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106604+0000", 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106604+0000", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106603+0000", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106608+0000", 2024-09-14T04:06:06.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6559999999999997e-06 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106618+0000", 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0093e-05 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106623+0000", 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1760000000000001e-06 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106692+0000", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8440000000000002e-05 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106773+0000", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1092000000000002e-05 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.106846+0000", 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2619000000000004e-05 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.110287+0000", 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034410759999999999 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.112699+0000", 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002412497 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.209817+0000", 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.097117815999999996 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.209863+0000", 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5877999999999997e-05 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.209899+0000", 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6288000000000002e-05 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.618 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194962 8.5 e116/98 8:a073886e:::benchmark_data_smithi001_125658_object69723:head v 116'12156, mlcod=116'12154)", 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.254941+0000", 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.30218071500002, 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45687488599999998, 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254941+0000", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254941+0000", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254940+0000", 2024-09-14T04:06:06.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254947+0000", 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9110000000000004e-06 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254959+0000", 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1488000000000001e-05 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.254965+0000", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5119999999999997e-06 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255007+0000", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1377e-05 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255026+0000", 2024-09-14T04:06:06.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9128e-05 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711764+0000", 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45673844800000002 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711816+0000", 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1857000000000002e-05 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194955 8.b e116/109 8:ddbfc7a9:::benchmark_data_smithi001_125658_object69716:head v 116'12413, mlcod=106'4827)", 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.255833+0000", 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.30128889700001, 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46103191700000001, 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255833+0000", 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255833+0000", 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255832+0000", 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255838+0000", 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6999999999999996e-06 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255846+0000", 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7579999999999992e-06 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255851+0000", 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.356e-06 2024-09-14T04:06:06.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255913+0000", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2002999999999996e-05 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.255934+0000", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0724999999999999e-05 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.716817+0000", 2024-09-14T04:06:06.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.460883349 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.716865+0000", 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7546000000000002e-05 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194967 8.4 e116/98 8:23d6bb2d:::benchmark_data_smithi001_125658_object69728:head v 116'12128, mlcod=116'12126)", 2024-09-14T04:06:06.628 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.258609+0000", 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.29851287100001, 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45323917699999999, 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258609+0000", 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258609+0000", 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258608+0000", 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258614+0000", 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0700000000000003e-06 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258625+0000", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0513000000000001e-05 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258631+0000", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2979999999999997e-06 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258685+0000", 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4382999999999998e-05 2024-09-14T04:06:06.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258707+0000", 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1367000000000001e-05 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711786+0000", 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45307897000000003 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711848+0000", 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2348999999999995e-05 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194966 8.e e116/102 8:7ed8c295:::benchmark_data_smithi001_125658_object69727:head v 116'12234, mlcod=116'12232)", 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.258708+0000", 2024-09-14T04:06:06.633 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.29841415499999, 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.457517171, 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258708+0000", 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258708+0000", 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258707+0000", 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258712+0000", 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4160000000000003e-06 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258721+0000", 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9809999999999992e-06 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258726+0000", 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9640000000000002e-06 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258778+0000", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1168000000000002e-05 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258796+0000", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8097e-05 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.716179+0000", 2024-09-14T04:06:06.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45738346400000002 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.716225+0000", 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5637999999999999e-05 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194956 8.b e116/109 8:d77d6ca8:::benchmark_data_smithi001_125658_object69717:head v 116'12414, mlcod=106'4827)", 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.262703+0000", 2024-09-14T04:06:06.638 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.294419248, 2024-09-14T04:06:06.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46514613199999999, 2024-09-14T04:06:06.639 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.639 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.262703+0000", 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.262703+0000", 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.262702+0000", 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.262709+0000", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2620000000000001e-06 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711715+0000", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.449006617 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711729+0000", 2024-09-14T04:06:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3387000000000001e-05 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711774+0000", 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5206000000000001e-05 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711808+0000", 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3880000000000001e-05 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.727754+0000", 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015945858 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.727849+0000", 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4845000000000004e-05 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34299.0:194975 8.e e116/102 8:757dee2f:::benchmark_data_smithi001_125658_object69736:head v 116'12235, mlcod=116'12232)", 2024-09-14T04:06:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.274488+0000", 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 360.28263339900002, 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46022992200000001, 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.274488+0000", 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.274488+0000", 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.274487+0000", 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.274497+0000", 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0516000000000001e-05 2024-09-14T04:06:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711710+0000", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.43721209900000002 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711729+0000", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9117999999999999e-05 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711789+0000", 2024-09-14T04:06:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0835999999999998e-05 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.711812+0000", 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2419999999999999e-05 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.734658+0000", 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022845721999999999 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.734718+0000", 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0689000000000001e-05 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:230013 8.2 8:4ad7c8eb:::benchmark_data_smithi001_125658_object104774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:49.820617+0000", 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 316.73650470299998, 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071243977, 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 230013 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820617+0000", 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820617+0000", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820616+0000", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820623+0000", 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4089999999999997e-06 2024-09-14T04:06:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820635+0000", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2320999999999999e-05 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820642+0000", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4330000000000001e-06 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820694+0000", 2024-09-14T04:06:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1839999999999998e-05 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820783+0000", 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9753000000000006e-05 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.820861+0000", 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7989000000000002e-05 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.824626+0000", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037642470000000001 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.824874+0000", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024801199999999998 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.891730+0000", 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066856129 2024-09-14T04:06:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.891820+0000", 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0671e-05 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:49.891861+0000", 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0710999999999997e-05 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:13475 9.c e139/135 9:3abfb4f8:::benchmark_data_smithi001_156540_object13474:head v 139'882, mlcod=139'880)", 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:04:19.673122+0000", 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.88399946, 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063113723999999996, 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673122+0000", 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673122+0000", 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673121+0000", 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673192+0000", 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1706999999999996e-05 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673209+0000", 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6460000000000002e-05 2024-09-14T04:06:06.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673217+0000", 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3620000000000006e-06 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673280+0000", 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2516999999999998e-05 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.673301+0000", 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0959e-05 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736194+0000", 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062893426000000002 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736236+0000", 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1909999999999997e-05 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:13481 9.c e139/135 9:30ff8afd:::benchmark_data_smithi001_156540_object13480:head v 139'883, mlcod=139'880)", 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:04:19.675503+0000", 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.881618695, 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060753970999999997, 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675503+0000", 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675503+0000", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675502+0000", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675549+0000", 2024-09-14T04:06:06.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7477e-05 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734156+0000", 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058606576000000001 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734169+0000", 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3713999999999999e-05 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734843+0000", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00067395900000000002 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734864+0000", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0931999999999999e-05 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736239+0000", 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013746100000000001 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736257+0000", 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8060999999999999e-05 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:13467 9.e e139/135 9:7c269b49:::benchmark_data_smithi001_156540_object13466:head v 139'840, mlcod=139'838)", 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:04:19.675873+0000", 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.881248531, 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060246260000000003, 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.665 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675873+0000", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675873+0000", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675872+0000", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.675961+0000", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8794000000000004e-05 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734160+0000", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058199213999999999 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734170+0000", 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9189999999999994e-06 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734334+0000", 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000163671 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.734353+0000", 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8932000000000001e-05 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736090+0000", 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017374619999999999 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:04:19.736120+0000", 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9349999999999999e-05 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:150159 9.c e146/135 9:3788b774:::benchmark_data_smithi001_156540_object18565:head v 146'9443, mlcod=146'9441)", 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:06.485641+0000", 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.071480999000000003, 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041530535, 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485641+0000", 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485641+0000", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485640+0000", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485650+0000", 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8369999999999992e-06 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485660+0000", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6520000000000001e-06 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.485666+0000", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4019999999999999e-06 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.498507+0000", 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012841406 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.498528+0000", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1092999999999999e-05 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.527007+0000", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028478799999999999 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.527171+0000", 2024-09-14T04:06:06.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016405099999999999 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:150160 9.4 e146/135 9:2330eba2:::benchmark_data_smithi001_156540_object18566:head v 146'9498, mlcod=146'9493)", 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:06.490251+0000", 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.066870737999999999, 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.036753319999999999, 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:06.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490251+0000", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490251+0000", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490249+0000", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:06.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490332+0000", 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2727999999999998e-05 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490352+0000", 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9794000000000001e-05 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.490360+0000", 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8789999999999992e-06 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.499022+0000", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086610439999999997 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.499046+0000", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4179e-05 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.526841+0000", 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027795299999999998 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:06.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:06.527004+0000", 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016336399999999999 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:06.678 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:06:06.694 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:06.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:06.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:06.939 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:06.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:06] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-14T04:06:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:06.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:07.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:07.103 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:07.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:07.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:07.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:07.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:06 smithi012 ceph-mon[101180]: pgmap v521: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:06:07.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:07.367 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:06 smithi001 ceph-mon[107399]: pgmap v521: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:06:07.367 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:06 smithi001 ceph-mon[112165]: pgmap v521: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:06:07.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:07.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:07.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:07.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:07.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:07.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:06:07.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:07.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:07.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:07.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:07.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:07.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:08.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:08.098 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:08.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:08.214 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:08.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:08.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:08.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:08.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:08.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:08.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:08.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:08.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.798 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:08.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:08.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:08.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:09.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:09.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:09.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:09.098 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:09.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:09.211 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:08 smithi001 ceph-mon[107399]: pgmap v522: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:09.211 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:08 smithi001 ceph-mon[112165]: pgmap v522: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:09.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:09.237 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:08 smithi012 ceph-mon[101180]: pgmap v522: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:09.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:09.379 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:09.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:09.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:09.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:09.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:09.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:09.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:09.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:09.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:09.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:09.927 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:09.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:09.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:10.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.115 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:10.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:10.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:10.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:10.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:10.399 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:06:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:06:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153519 9.f 9:f56e612e:::benchmark_data_smithi001_156540_object21925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.382267+0000", 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014186471000000001, 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014217479, 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153519 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382267+0000", 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382267+0000", 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382265+0000", 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382271+0000", 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7259999999999999e-06 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382282+0000", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1080999999999999e-05 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382289+0000", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5540000000000003e-06 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382408+0000", 2024-09-14T04:06:10.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119777 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382526+0000", 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117751 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382642+0000", 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115958 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394812+0000", 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012169831000000001 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394908+0000", 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6215e-05 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.405 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153521 9.7 9:e2b5b182:::benchmark_data_smithi001_156540_object21927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.382333+0000", 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.01412069, 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014240777, 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.406 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153521 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382333+0000", 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382333+0000", 2024-09-14T04:06:10.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382332+0000", 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382335+0000", 2024-09-14T04:06:10.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8720000000000002e-06 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382341+0000", 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.981e-06 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.382346+0000", 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.118e-06 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387360+0000", 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050140699999999998 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387461+0000", 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100926 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:06:10.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387557+0000", 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5550000000000005e-05 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394817+0000", 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072604150000000001 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394849+0000", 2024-09-14T04:06:10.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1985e-05 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153524 9.f 9:f3b534ce:::benchmark_data_smithi001_156540_object21930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.387824+0000", 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0086290059999999998, 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0088302060000000002, 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.412 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153524 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387824+0000", 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387824+0000", 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387823+0000", 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387828+0000", 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7160000000000002e-06 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387839+0000", 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1863e-05 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.387846+0000", 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7259999999999996e-06 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.389009+0000", 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011627740000000001 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.389144+0000", 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135241 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.389278+0000", 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013362000000000001 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394856+0000", 2024-09-14T04:06:10.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055777140000000001 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153526 9.7 9:eb68a67a:::benchmark_data_smithi001_156540_object21932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.388545+0000", 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0079087789999999995, 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0081942290000000008, 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.418 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153526 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388545+0000", 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388545+0000", 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388544+0000", 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388549+0000", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1939999999999997e-06 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388560+0000", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0793e-05 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388566+0000", 2024-09-14T04:06:10.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.472e-06 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395061+0000", 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064951080000000003 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395264+0000", 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020232799999999999 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395384+0000", 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012010600000000001 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153530 9.6 9:6342e348:::benchmark_data_smithi001_156540_object21936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.390931+0000", 2024-09-14T04:06:10.423 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0055221929999999999, 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058762889999999998, 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153530 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390931+0000", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390931+0000", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390930+0000", 2024-09-14T04:06:10.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390936+0000", 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1619999999999996e-06 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390947+0000", 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0930999999999999e-05 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.390954+0000", 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0310000000000001e-06 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.391004+0000", 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9631e-05 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.391102+0000", 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8336000000000002e-05 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.391196+0000", 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3943000000000005e-05 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:153525 9.5 e146/135 9:a939ca82:::benchmark_data_smithi001_156540_object21931:head v 146'9630, mlcod=146'9628)", 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.393989+0000", 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0024643619999999999, 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028860449999999998, 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.393989+0000", 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.393989+0000", 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.393988+0000", 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:10.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.393998+0000", 2024-09-14T04:06:10.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0028e-05 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394013+0000", 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4284000000000001e-05 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394019+0000", 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6359999999999997e-06 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394075+0000", 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5340999999999998e-05 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.394095+0000", 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0412e-05 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.435 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153533 9.a 9:5b3d09aa:::benchmark_data_smithi001_156540_object21939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.435 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.395166+0000", 2024-09-14T04:06:10.435 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001286976, 2024-09-14T04:06:10.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017672429999999999, 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153533 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395166+0000", 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395166+0000", 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395165+0000", 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395171+0000", 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3849999999999999e-06 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395182+0000", 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0377000000000001e-05 2024-09-14T04:06:10.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395189+0000", 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2799999999999998e-06 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395242+0000", 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3273000000000003e-05 2024-09-14T04:06:10.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395359+0000", 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117049 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.395463+0000", 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001041 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.440 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:153513 9.6 9:6c6b883f:::benchmark_data_smithi001_156540_object21919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:10.375481+0000", 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.020972319999999999, 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021521004, 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:10.441 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153513 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375481+0000", 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375481+0000", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375479+0000", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375485+0000", 2024-09-14T04:06:10.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1489999999999999e-06 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375500+0000", 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4426999999999999e-05 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.375507+0000", 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3189999999999997e-06 2024-09-14T04:06:10.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.383615+0000", 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0081081690000000001 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.383723+0000", 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107398 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.383820+0000", 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7046999999999994e-05 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.388416+0000", 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045965210000000001 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:06:10.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:10.396398+0000", 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079820359999999996 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 8 2024-09-14T04:06:10.447 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:06:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:10.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:10.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:10.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:10.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:10.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:10.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:10.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:10.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:10.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:10.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:11.004 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:06:11.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:11.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:11.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:10 smithi001 ceph-mon[112165]: pgmap v523: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:11.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:10 smithi001 ceph-mon[107399]: pgmap v523: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:11.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:11.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:11.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:11.339 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:10 smithi012 ceph-mon[101180]: pgmap v523: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:06:11.339 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:11.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:06:11.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:11.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:11.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:11.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:11.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:11.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:11.740 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:11.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:11.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:11.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:11.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:11.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:11.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:12.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:12.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:12.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:12.279 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:12.281 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:12.281 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:12.281 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[112165]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T04:06:12.282 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:12.282 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:12.282 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:11 smithi001 ceph-mon[107399]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T04:06:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:11 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2656569443' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:11 smithi012 ceph-mon[101180]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T04:06:12.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:12.420 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:12.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:12.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:12.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:12.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:12.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:12.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:12.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:12.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:12.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:12.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:13.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:13 smithi012 ceph-mon[101180]: pgmap v525: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:06:13.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:13.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:13.216 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:13.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:13 smithi001 ceph-mon[107399]: pgmap v525: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:06:13.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:13 smithi001 ceph-mon[112165]: pgmap v525: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:06:13.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:13.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:13.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:13.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:13.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:13.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:13.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:13.641 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:06:13.641 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:06:13.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:13.675 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:13.676 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:13.706 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:13.706 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:13.740 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:13.740 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:13.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:13.774 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:13.774 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:13.810 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:13.811 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:13.811 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:13.844 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:13.844 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:13.875 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:13.875 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:13.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:13.942 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:13.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:14.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:14.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:14.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:14.203 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1956475742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:14.203 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1956475742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:14.223 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:14.312 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1956475742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:14.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:14.359 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:14.359 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:14.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:14.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:14.517 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:14.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:14.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:14.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:14.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:14.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:14.897 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:14.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:14.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:15.021 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:06:14.735+0000 7fbfffcba640 -1 mon.a@0(leader).osd e147 definitely_dead 0 2024-09-14T04:06:15.029 INFO:teuthology.orchestra.run.smithi001.stderr:marked out osd.3. 2024-09-14T04:06:15.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:15.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:15.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:06:15.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:15.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:15.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:15.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:15.335 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:15 smithi012 ceph-mon[101180]: pgmap v526: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:06:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-14T04:06:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:15 smithi012 ceph-mon[101180]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-14T04:06:15.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[112165]: pgmap v526: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:06:15.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-14T04:06:15.353 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[112165]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-14T04:06:15.353 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[107399]: pgmap v526: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:06:15.353 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-14T04:06:15.353 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:15 smithi001 ceph-mon[107399]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-14T04:06:15.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:06:15.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:15.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:15.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:15.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:15.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:15.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:15.774 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:15.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:15.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:15.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:15.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:15.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:16.036 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-14T04:06:16.036 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:16 smithi012 ceph-mon[101180]: osdmap e148: 8 total, 8 up, 7 in 2024-09-14T04:06:16.036 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:16.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:16.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:16.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:16.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:16.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:16.339 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:16.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-14T04:06:16.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[107399]: osdmap e148: 8 total, 8 up, 7 in 2024-09-14T04:06:16.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:16.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1576584765' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-14T04:06:16.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[112165]: osdmap e148: 8 total, 8 up, 7 in 2024-09-14T04:06:16.341 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:16.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:16.464 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:16.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:16.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:16.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:16.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:16.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:06:16.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:16.875 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:16.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:16.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:16] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-14T04:06:16.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:17.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:17.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:17.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:17.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:17.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:17.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:17 smithi012 ceph-mon[101180]: pgmap v528: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 617 op/s 2024-09-14T04:06:17.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:17 smithi012 ceph-mon[101180]: osdmap e149: 8 total, 8 up, 7 in 2024-09-14T04:06:17.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:17.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:17.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:17 smithi001 ceph-mon[107399]: pgmap v528: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 617 op/s 2024-09-14T04:06:17.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:17 smithi001 ceph-mon[107399]: osdmap e149: 8 total, 8 up, 7 in 2024-09-14T04:06:17.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:17 smithi001 ceph-mon[112165]: pgmap v528: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 617 op/s 2024-09-14T04:06:17.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:17 smithi001 ceph-mon[112165]: osdmap e149: 8 total, 8 up, 7 in 2024-09-14T04:06:17.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:17.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:17.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:17.726 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:17.828 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:17.864 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:17.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:17.966 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:18.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:18.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:18.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:18.211 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:18.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:18.329 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9748 2024-09-14T04:06:18.329 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 147'9955 2024-09-14T04:06:18.329 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 147'9863 2024-09-14T04:06:18.329 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 147'9979 2024-09-14T04:06:18.330 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: osdmap e150: 8 total, 8 up, 7 in 2024-09-14T04:06:18.330 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9935 2024-09-14T04:06:18.330 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 147'9892 2024-09-14T04:06:18.330 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 147'9901 2024-09-14T04:06:18.330 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:18 smithi012 ceph-mon[101180]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 147'9847 2024-09-14T04:06:18.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:18.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:18.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9748 2024-09-14T04:06:18.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 147'9955 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 147'9863 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 147'9979 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: osdmap e150: 8 total, 8 up, 7 in 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9935 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 147'9892 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 147'9901 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[107399]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 147'9847 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9748 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 147'9955 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 147'9863 2024-09-14T04:06:18.364 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 147'9979 2024-09-14T04:06:18.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: osdmap e150: 8 total, 8 up, 7 in 2024-09-14T04:06:18.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 147'9935 2024-09-14T04:06:18.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 147'9892 2024-09-14T04:06:18.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 147'9901 2024-09-14T04:06:18.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:18 smithi001 ceph-mon[112165]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 147'9847 2024-09-14T04:06:18.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:18.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:18.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:18.655 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:18.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:18.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:18.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:18.957 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:18.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:19.049 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:06:19.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:19.067 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:19.068 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:19.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:19.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:19.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:19.339 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:19 smithi012 ceph-mon[101180]: pgmap v531: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:19 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:06:19.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:19.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:19.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[112165]: pgmap v531: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:19.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:19.407 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:06:19.407 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[107399]: pgmap v531: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:19.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:19.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:06:19.475 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:19.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:19.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:19.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:19.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:19.758 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:06:19.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:06:19.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:19.928 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:19.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:20.030 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:20.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:06:20.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:20.142 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:20.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:06:20.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[107399]: osdmap e151: 8 total, 8 up, 7 in 2024-09-14T04:06:20.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:20.143 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:20.144 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:20.144 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:06:20.144 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[112165]: osdmap e151: 8 total, 8 up, 7 in 2024-09-14T04:06:20.144 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:20.144 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:20.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:20.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:20.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:20 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:20.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3086294264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:06:20.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:20 smithi012 ceph-mon[101180]: osdmap e151: 8 total, 8 up, 7 in 2024-09-14T04:06:20.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:20.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:20.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:20.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:20.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:20.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:20.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:20.648 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:20.695 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:20.695 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:06:20.695 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:20.695 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-14T04:06:20.696 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0 2024-09-14T04:06:20.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:20.799 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:20.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:20.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:20.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:20.966 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:20.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:20.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:20.985 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:20 smithi001 systemd[1]: Stopping Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:21.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:21.109 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:21.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.272 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:21.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:21.338 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:21 smithi001 ceph-mon[107399]: pgmap v533: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:21.338 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:21 smithi001 ceph-mon[112165]: pgmap v533: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:21.343 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:21 smithi012 ceph-mon[101180]: pgmap v533: 89 pgs: 1 remapped+peering, 3 unknown, 18 peering, 67 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:06:21.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:21.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:21.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:21.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:21.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:21.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:21.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:21.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:21.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:21.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:06:21.962 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:06:21.966 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:06:21.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:22.024 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:22.024 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:22.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:22.058 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:22.058 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:22.097 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:22.097 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:22.139 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:22.140 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:22.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:22.175 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:22.176 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:22.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:22.209 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3323159182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:22.209 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-mon[112165]: osd.0 marked itself down and dead 2024-09-14T04:06:22.209 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:22.210 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:22.210 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3323159182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:22.210 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-mon[107399]: osd.0 marked itself down and dead 2024-09-14T04:06:22.210 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T04:06:22.068+0000 7f80fd049640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:22.210 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T04:06:22.068+0000 7f80fd049640 -1 osd.0 151 *** Got signal Terminated *** 2024-09-14T04:06:22.210 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[123832]: 2024-09-14T04:06:22.068+0000 7f80fd049640 -1 osd.0 151 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:06:22.234 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:22.234 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:22.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:22.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:22 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3323159182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:22 smithi012 ceph-mon[101180]: osd.0 marked itself down and dead 2024-09-14T04:06:22.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:22.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:22.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:22.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:22.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:22.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:22.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:22.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:22.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:22.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:22.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:22.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:23.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.115 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:23.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:23.216 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:23.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:23.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:23.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[107399]: pgmap v534: 89 pgs: 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 1 remapped+peering, 81 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 43644/310707 objects misplaced (14.047%); 15 MiB/s, 248 objects/s recovering 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[107399]: osdmap e152: 8 total, 7 up, 7 in 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[112165]: pgmap v534: 89 pgs: 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 1 remapped+peering, 81 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 43644/310707 objects misplaced (14.047%); 15 MiB/s, 248 objects/s recovering 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:23 smithi001 ceph-mon[112165]: osdmap e152: 8 total, 7 up, 7 in 2024-09-14T04:06:23.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:23.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:23.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:23 smithi012 ceph-mon[101180]: pgmap v534: 89 pgs: 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 1 remapped+peering, 81 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 43644/310707 objects misplaced (14.047%); 15 MiB/s, 248 objects/s recovering 2024-09-14T04:06:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:23 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2024-09-14T04:06:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:23 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:23 smithi012 ceph-mon[101180]: osdmap e152: 8 total, 7 up, 7 in 2024-09-14T04:06:23.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.672 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:23.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:23.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:23.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:23.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:23.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.020 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:24.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:24.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:24.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:24.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:24.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:24.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:24.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:24.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:24.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:24 smithi001 ceph-mon[112165]: osdmap e153: 8 total, 7 up, 7 in 2024-09-14T04:06:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:24 smithi001 ceph-mon[112165]: 9.8 continuing backfill to osd.7 from (140'2141,151'10130] 9:10016cea:::benchmark_data_smithi001_156540_object65112:head to 151'10130 2024-09-14T04:06:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:24 smithi001 ceph-mon[107399]: osdmap e153: 8 total, 7 up, 7 in 2024-09-14T04:06:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:24 smithi001 ceph-mon[107399]: 9.8 continuing backfill to osd.7 from (140'2141,151'10130] 9:10016cea:::benchmark_data_smithi001_156540_object65112:head to 151'10130 2024-09-14T04:06:24.529 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:24.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:24 smithi012 ceph-mon[101180]: osdmap e153: 8 total, 7 up, 7 in 2024-09-14T04:06:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:24 smithi012 ceph-mon[101180]: 9.8 continuing backfill to osd.7 from (140'2141,151'10130] 9:10016cea:::benchmark_data_smithi001_156540_object65112:head to 151'10130 2024-09-14T04:06:24.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:24.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:24.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:24.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:24.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:24.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:24.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:25.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:25.103 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:25.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:25.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:25.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:25.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:25.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:25.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:25.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:25 smithi001 ceph-mon[112165]: pgmap v537: 89 pgs: 4 peering, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 60 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 50097/308286 objects misplaced (16.250%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:06:25.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:25 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:06:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:25 smithi001 ceph-mon[107399]: pgmap v537: 89 pgs: 4 peering, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 60 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 50097/308286 objects misplaced (16.250%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:06:25.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:25 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:06:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.509 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:25.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:25 smithi012 ceph-mon[101180]: pgmap v537: 89 pgs: 4 peering, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 2 active+remapped+backfill_wait, 5 active+remapped+backfilling, 60 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 50097/308286 objects misplaced (16.250%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:06:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:25 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:06:25.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:25.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:25.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:25.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:25.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:25.924 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:25.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:25.971 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:25 smithi001 podman[171235]: 2024-09-14 04:06:25.625190883 +0000 UTC m=+3.705562793 container died 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-14T04:06:26.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:26.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:26.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:26.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:26.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:26.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:26.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:26.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:26.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:26 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 1/306699 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:06:26.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPushReply(9.1 153/149 [PushReplyOp(9:8402077f:::benchmark_data_smithi001_156540_object64882:head)])", 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:26.431506+0000", 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0013956999999999999, 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00142256, 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:26.435 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431506+0000", 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431506+0000", 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431506+0000", 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431511+0000", 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5800000000000002e-06 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:26.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431519+0000", 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3059999999999998e-06 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.431527+0000", 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0430000000000008e-06 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:163855 9.0 e153/135 9:08a74db3:::benchmark_data_smithi001_156540_object32261:head v 153'10142, mlcod=153'10140)", 2024-09-14T04:06:26.438 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:26.432486+0000", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00041621600000000001, 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00049825800000000001, 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432486+0000", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432486+0000", 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432485+0000", 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432492+0000", 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9739999999999996e-06 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:26.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432506+0000", 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4368e-05 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432513+0000", 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2729999999999999e-06 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432562+0000", 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9257e-05 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432582+0000", 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9811000000000001e-05 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:163856 9.1 9:84307007:::benchmark_data_smithi001_156540_object32262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:26.432521+0000", 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00038078800000000001, 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00052658900000000005, 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:26.442 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 163856 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432521+0000", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432521+0000", 2024-09-14T04:06:26.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432520+0000", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432526+0000", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4559999999999997e-06 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432539+0000", 2024-09-14T04:06:26.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3222e-05 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432546+0000", 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9419999999999998e-06 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432739+0000", 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019286999999999999 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:06:26.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.432886+0000", 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014692399999999999 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:26.433028+0000", 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014152199999999999 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2024-09-14T04:06:26.446 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:06:26.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:26 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 1/306699 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:06:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:26 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 1/306699 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:06:26.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:26 smithi001 podman[171235]: 2024-09-14 04:06:26.370871828 +0000 UTC m=+4.451243730 container cleanup 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:06:26.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:26 smithi001 bash[171235]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0 2024-09-14T04:06:26.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:26.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:26.662 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:26.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:26.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:26.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:26.849 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:26] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.51.0" 2024-09-14T04:06:26.850 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:26 smithi001 podman[171441]: 2024-09-14 04:06:26.597948443 +0000 UTC m=+3.827280119 container remove 6ed72c8797bab2f1b75d5d546447f5c74cc1b8e72eed45d3747b2c15ae012dfc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-14T04:06:26.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:26.970 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:26.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:27.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:27.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:27.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.263 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:27.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:27.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:27 smithi012 ceph-mon[101180]: pgmap v538: 89 pgs: 1 active+clean+remapped, 4 peering, 1 stale+active+remapped+backfill_wait, 12 stale+active+clean, 6 active+undersized, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 active+undersized+degraded, 58 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/306699 objects degraded (0.000%); 48936/306699 objects misplaced (15.956%); 23 MiB/s, 367 objects/s recovering 2024-09-14T04:06:27.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:27.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:27 smithi001 ceph-mon[112165]: pgmap v538: 89 pgs: 1 active+clean+remapped, 4 peering, 1 stale+active+remapped+backfill_wait, 12 stale+active+clean, 6 active+undersized, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 active+undersized+degraded, 58 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/306699 objects degraded (0.000%); 48936/306699 objects misplaced (15.956%); 23 MiB/s, 367 objects/s recovering 2024-09-14T04:06:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:27 smithi001 ceph-mon[107399]: pgmap v538: 89 pgs: 1 active+clean+remapped, 4 peering, 1 stale+active+remapped+backfill_wait, 12 stale+active+clean, 6 active+undersized, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 active+undersized+degraded, 58 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/306699 objects degraded (0.000%); 48936/306699 objects misplaced (15.956%); 23 MiB/s, 367 objects/s recovering 2024-09-14T04:06:27.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:06:27.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:27.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:27.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:06:27.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:27.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:27.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:27.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:27.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:27.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:27.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:28.074 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:28.076 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:28.100 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:28.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:28.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:28.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:28.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:28 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:06:28.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:28 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:06:28.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:28.504 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:28 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:06:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:28.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:28.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:28.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:28.840 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:28.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:28.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:28.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:29.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:29.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:29.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:29.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:29.301 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:29.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:29.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:29.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:29.452 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:29 smithi001 ceph-mon[107399]: pgmap v539: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 39 MiB/s, 620 objects/s recovering 2024-09-14T04:06:29.452 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:29 smithi001 ceph-mon[112165]: pgmap v539: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 39 MiB/s, 620 objects/s recovering 2024-09-14T04:06:29.469 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:29.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:29.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:29 smithi012 ceph-mon[101180]: pgmap v539: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 39 MiB/s, 620 objects/s recovering 2024-09-14T04:06:29.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:29.732 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:29.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:29.834 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:29.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:29.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:29.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:30.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:30.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:30.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:30.225 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:06:30.245 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:30.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:30.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:30.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:30.496 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:30.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:30.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:30.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:30.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:30.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:30.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:30.939 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:30.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:31.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:31.061 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:31.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:31.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:31.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:31.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:31.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:31.466 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[112165]: pgmap v540: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 27 MiB/s, 433 objects/s recovering 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[112165]: osdmap e154: 8 total, 7 up, 7 in 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[107399]: pgmap v540: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 27 MiB/s, 433 objects/s recovering 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:06:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:31.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:31 smithi001 ceph-mon[107399]: osdmap e154: 8 total, 7 up, 7 in 2024-09-14T04:06:31.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:31 smithi012 ceph-mon[101180]: pgmap v540: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 24735/300285 objects degraded (8.237%); 45992/300285 objects misplaced (15.316%); 27 MiB/s, 433 objects/s recovering 2024-09-14T04:06:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:31 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:06:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2550058690' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:31.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:31 smithi012 ceph-mon[101180]: osdmap e154: 8 total, 7 up, 7 in 2024-09-14T04:06:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.607 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:31.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:31.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:31.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:31.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:31.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:31.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:32.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:32.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:32.141 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:31 smithi001 podman[172281]: 2024-09-14 04:06:31.719887879 +0000 UTC m=+1.158733449 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:06:32.141 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:31 smithi001 podman[172281]: 2024-09-14 04:06:31.853330087 +0000 UTC m=+1.292175643 container create b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231) 2024-09-14T04:06:32.141 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:32 smithi001 podman[172281]: 2024-09-14 04:06:32.140170664 +0000 UTC m=+1.579016218 container init b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:06:32.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:32.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.281 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:32.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:32.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:32.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:32.449 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:32 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 23369/288837 objects degraded (8.091%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:32.449 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:32 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 23369/288837 objects degraded (8.091%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:32.449 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:32 smithi001 podman[172281]: 2024-09-14 04:06:32.146331228 +0000 UTC m=+1.585176782 container start b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T04:06:32.449 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:32 smithi001 podman[172281]: 2024-09-14 04:06:32.240370312 +0000 UTC m=+1.679215869 container attach b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T04:06:32.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:32.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:32 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 23369/288837 objects degraded (8.091%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:32.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:32.674 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:32.704 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:32 smithi001 conmon[172392]: conmon b34ba984c512fda716de : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8.scope/memory.events 2024-09-14T04:06:32.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:32.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:32.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:32.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:32.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:33.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:33.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:33.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:33.212 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:33.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:06:33.285 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:06:32 smithi001 podman[172281]: 2024-09-14 04:06:32.980206234 +0000 UTC m=+2.419051802 container died b34ba984c512fda716de6a6b9716bece4ffa48c812caddb558a93067e50577d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T04:06:33.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:33.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:33.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:33.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:33.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:33.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:33 smithi012 ceph-mon[101180]: pgmap v542: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 37 MiB/s, 587 objects/s recovering 2024-09-14T04:06:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.638 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:33.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:33 smithi001 ceph-mon[112165]: pgmap v542: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 37 MiB/s, 587 objects/s recovering 2024-09-14T04:06:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:33 smithi001 ceph-mon[107399]: pgmap v542: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 37 MiB/s, 587 objects/s recovering 2024-09-14T04:06:33.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:33.762 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:33.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service' 2024-09-14T04:06:33.864 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:33.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:33.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:33.960 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:33.960 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-14T04:06:34.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:34.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:34.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:34.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:34.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:34.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:34.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.458 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:34.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:34.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:34.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:34.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:34.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:34.836 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:34.837 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:34.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:34.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:34.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:34.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:35.038 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:35.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:35.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:35.212 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:35.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.364 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:35.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:35.445 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: pgmap v543: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 30 MiB/s, 484 objects/s recovering 2024-09-14T04:06:35.446 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1980852946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:35.446 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.446 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.446 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.446 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:35.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: pgmap v543: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 30 MiB/s, 484 objects/s recovering 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1980852946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: pgmap v543: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 23369/288837 objects degraded (8.091%); 43292/288837 objects misplaced (14.988%); 30 MiB/s, 484 objects/s recovering 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1980852946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:35.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:35.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:35.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:35.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:35.619 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:35.623 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:35.646 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:06:35.647 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:06:35.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:35.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:35.854 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:35.898 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:35.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:35.912 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:35.912 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:36.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:36.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:36.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:36.136 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:36.186 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:36.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:36.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:36.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:36.465 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:36.567 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:36.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:36.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:06:36.696 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:36.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:36.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:36.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:36.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:36.904 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:36] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-14T04:06:36.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:37.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:37.108 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:06:37.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:37.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:37.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:37.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:37.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:37.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:37.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:37.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:37 smithi012 ceph-mon[101180]: pgmap v544: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 23369/286794 objects degraded (8.148%); 42859/286794 objects misplaced (14.944%); 25 MiB/s, 403 objects/s recovering 2024-09-14T04:06:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.606 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:37.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[107399]: pgmap v544: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 23369/286794 objects degraded (8.148%); 42859/286794 objects misplaced (14.944%); 25 MiB/s, 403 objects/s recovering 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[112165]: pgmap v544: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 23369/286794 objects degraded (8.148%); 42859/286794 objects misplaced (14.944%); 25 MiB/s, 403 objects/s recovering 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:37.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.760 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:37.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:37.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:37.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:37.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:37.965 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:37.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:38.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:38.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:38.169 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:38.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:38.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:38.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:38.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1554279582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:38.421 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:38.422 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:38.422 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1554279582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.422 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:38 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.451 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:38.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1554279582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:38 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:06:38.603 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:38.697 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:06:38.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:38.747 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:38.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:38.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:38.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:38.960 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:38.961 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:06:38.961 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-09-14T04:06:38.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd in 3 2024-09-14T04:06:38.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:39.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:39.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:39.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:39.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:39.308 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:39.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:39.410 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:39.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:39.484 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:39.484 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:39.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:39.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: pgmap v545: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 17 MiB/s, 268 objects/s recovering 2024-09-14T04:06:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: pgmap v546: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 20 MiB/s, 322 objects/s recovering 2024-09-14T04:06:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-14T04:06:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 22101/277776 objects degraded (7.956%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:39.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:39.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:39 smithi012 ceph-mon[101180]: osdmap e155: 8 total, 7 up, 7 in 2024-09-14T04:06:39.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:39.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: pgmap v545: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 17 MiB/s, 268 objects/s recovering 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: pgmap v546: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 20 MiB/s, 322 objects/s recovering 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 22101/277776 objects degraded (7.956%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[107399]: osdmap e155: 8 total, 7 up, 7 in 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: pgmap v545: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 17 MiB/s, 268 objects/s recovering 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: pgmap v546: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 22101/277776 objects degraded (7.956%); 40781/277776 objects misplaced (14.681%); 20 MiB/s, 322 objects/s recovering 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 22101/277776 objects degraded (7.956%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:06:39.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:39 smithi001 ceph-mon[112165]: osdmap e155: 8 total, 7 up, 7 in 2024-09-14T04:06:39.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:39.849 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:39.951 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:39.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:39.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:40.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:40.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.123 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:40.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:40.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:40.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:06:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:40.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:40.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:40.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:40.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:40.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:40.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:40.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:40.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:40.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:40.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:40.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:40.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:40.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:40.988 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:41.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:41.123 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:41.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:41.250 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:06:41.263 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:41.285 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:41.285 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:41.319 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:41.319 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:41.345 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:41.346 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:41.365 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:41.377 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:41.378 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:41.402 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:41.410 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:41.410 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:41.445 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:41.445 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:41.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:41 smithi001 ceph-mon[112165]: pgmap v548: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 22101/275610 objects degraded (8.019%); 40327/275610 objects misplaced (14.632%); 9.8 MiB/s, 156 objects/s recovering 2024-09-14T04:06:41.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1513246029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:41.478 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:41.478 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:41.479 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:41 smithi001 ceph-mon[107399]: pgmap v548: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 22101/275610 objects degraded (8.019%); 40327/275610 objects misplaced (14.632%); 9.8 MiB/s, 156 objects/s recovering 2024-09-14T04:06:41.479 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1513246029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:41.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:41.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:41 smithi012 ceph-mon[101180]: pgmap v548: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 22101/275610 objects degraded (8.019%); 40327/275610 objects misplaced (14.632%); 9.8 MiB/s, 156 objects/s recovering 2024-09-14T04:06:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1513246029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:41.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:41.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:41.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:41.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:41.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:06:41.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:41.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:41.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:41.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:42.063 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:42.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:42.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:42.244 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:42.346 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:42.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:42.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:42.524 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:42.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:42.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.671 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:42.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:42.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:42.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:42.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:42.899 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:42.934 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:42.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:43.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:43.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:43.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:43.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:43.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:43.348 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:43.356 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:43.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:43.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:43.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:43 smithi012 ceph-mon[101180]: pgmap v549: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:43.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:43 smithi001 ceph-mon[112165]: pgmap v549: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:43 smithi001 ceph-mon[107399]: pgmap v549: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:43.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:43.772 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:43.874 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:43.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:43.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:44.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:44.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:44.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:44.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:44.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:44.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:44.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:44.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:44.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 20774/266322 objects degraded (7.800%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 20774/266322 objects degraded (7.800%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:44.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:44.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:44.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 20774/266322 objects degraded (7.800%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:44.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:44.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:44.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:44.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:44.907 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:44.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:45.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:45.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:45.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:45.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:45.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:45.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:45.330 INFO:teuthology.orchestra.run.smithi001.stderr:marked in osd.3. 2024-09-14T04:06:45.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:45.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.456 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:45.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:45.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[112165]: pgmap v550: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:45.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3373331831' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:06:45.171+0000 7fbfffcba640 -1 mon.a@0(leader).osd e155 definitely_dead 0 2024-09-14T04:06:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[107399]: pgmap v550: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3373331831' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:45 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:45 smithi012 ceph-mon[101180]: pgmap v550: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 22 active+undersized, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 13 active+undersized+degraded, 46 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 20774/266322 objects degraded (7.800%); 38306/266322 objects misplaced (14.383%); 19 MiB/s, 302 objects/s recovering 2024-09-14T04:06:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3373331831' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:45 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-14T04:06:45.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:45.704 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:06:45.730 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:06:45.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:45.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:45.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:45.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:45.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:45.997 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:46.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:46.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:46.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:46.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:46.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:46.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:46.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[107399]: osdmap e156: 8 total, 7 up, 8 in 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[107399]: osdmap e157: 8 total, 7 up, 8 in 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[112165]: osdmap e156: 8 total, 7 up, 8 in 2024-09-14T04:06:46.447 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-mon[112165]: osdmap e157: 8 total, 7 up, 8 in 2024-09-14T04:06:46.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:46.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:46.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-14T04:06:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:46 smithi012 ceph-mon[101180]: osdmap e156: 8 total, 7 up, 8 in 2024-09-14T04:06:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:46 smithi012 ceph-mon[101180]: osdmap e157: 8 total, 7 up, 8 in 2024-09-14T04:06:46.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:46.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:46.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:46.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:46.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:46.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:46] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-14T04:06:47.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:47.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:47.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:47.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:47.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:47.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:47.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:47.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:47.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:47 smithi012 ceph-mon[101180]: pgmap v552: 89 pgs: 3 remapped+peering, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 18 active+undersized, 3 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 20284/258366 objects degraded (7.851%); 25883/258366 objects misplaced (10.018%); 16 MiB/s, 261 objects/s recovering 2024-09-14T04:06:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:47 smithi012 ceph-mon[101180]: osdmap e158: 8 total, 7 up, 8 in 2024-09-14T04:06:47.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:47.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:47.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:47.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:47 smithi001 ceph-mon[112165]: pgmap v552: 89 pgs: 3 remapped+peering, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 18 active+undersized, 3 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 20284/258366 objects degraded (7.851%); 25883/258366 objects misplaced (10.018%); 16 MiB/s, 261 objects/s recovering 2024-09-14T04:06:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:47 smithi001 ceph-mon[112165]: osdmap e158: 8 total, 7 up, 8 in 2024-09-14T04:06:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:47 smithi001 ceph-mon[107399]: pgmap v552: 89 pgs: 3 remapped+peering, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 18 active+undersized, 3 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 20284/258366 objects degraded (7.851%); 25883/258366 objects misplaced (10.018%); 16 MiB/s, 261 objects/s recovering 2024-09-14T04:06:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:47 smithi001 ceph-mon[107399]: osdmap e158: 8 total, 7 up, 8 in 2024-09-14T04:06:47.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:06:47.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:47.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:47.906 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:47.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:48.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:48.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:48.114 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:06:48.114 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:06:48.114 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179276 9.2 e158/157 9:48bcf8f1:::benchmark_data_smithi001_156540_object47682:head v 158'11109, mlcod=158'11106)", 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.103490+0000", 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0070186800000000002, 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0070475980000000004, 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.115 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103490+0000", 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103490+0000", 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103491+0000", 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.54e-07 2024-09-14T04:06:48.116 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103507+0000", 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6016e-05 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103526+0000", 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8372e-05 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.117 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103534+0000", 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7830000000000004e-06 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103778+0000", 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024403399999999999 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103798+0000", 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9896000000000001e-05 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:179280 9.0 9:0af0ea07:::benchmark_data_smithi001_156540_object47686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.103991+0000", 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0065177849999999999, 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006609607, 2024-09-14T04:06:48.119 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 179280 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.120 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103991+0000", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103991+0000", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103993+0000", 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7680000000000001e-06 2024-09-14T04:06:48.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.103998+0000", 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4499999999999997e-06 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104010+0000", 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2061e-05 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.122 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104028+0000", 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7904e-05 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104138+0000", 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000110037 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104265+0000", 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000127551 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104401+0000", 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013580299999999999 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179279 9.a e158/135 9:5f01f253:::benchmark_data_smithi001_156540_object47685:head v 158'11123, mlcod=158'11121)", 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.104334+0000", 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0061750060000000002, 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063342490000000001, 2024-09-14T04:06:48.124 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104334+0000", 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104334+0000", 2024-09-14T04:06:48.125 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104335+0000", 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4199999999999995e-07 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104341+0000", 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7479999999999999e-06 2024-09-14T04:06:48.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104353+0000", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.216e-05 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104369+0000", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6660999999999999e-05 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110244+0000", 2024-09-14T04:06:48.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058748769999999997 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110264+0000", 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9616e-05 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179277 9.2 e158/157)", 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.104717+0000", 2024-09-14T04:06:48.128 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0057918240000000001, 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059988150000000002, 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104717+0000", 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.129 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104717+0000", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104718+0000", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6100000000000001e-07 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104723+0000", 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7230000000000004e-06 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104730+0000", 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4919999999999996e-06 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.104736+0000", 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5570000000000003e-06 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:179286 9.0 9:02fd8fc8:::benchmark_data_smithi001_156540_object47692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.106002+0000", 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0045068349999999998, 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047589310000000001, 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:48.132 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 179286 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106002+0000", 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106002+0000", 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106004+0000", 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8500000000000001e-06 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106010+0000", 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3709999999999999e-06 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106027+0000", 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7467000000000001e-05 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106035+0000", 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8429999999999993e-06 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108170+0000", 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021356560000000001 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108358+0000", 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018758499999999999 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108478+0000", 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119518 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.136 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179284 9.3 e158/157 9:cb309b7c:::benchmark_data_smithi001_156540_object47690:head v 158'11215, mlcod=158'11213)", 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.106638+0000", 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.003870999, 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004193039, 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106638+0000", 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106638+0000", 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106639+0000", 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4600000000000003e-07 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106646+0000", 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2200000000000003e-06 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106657+0000", 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1189000000000001e-05 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106664+0000", 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2820000000000002e-06 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.107474+0000", 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00081059000000000003 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.107493+0000", 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8244e-05 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.140 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179283 9.d e158/135 9:b47fa56f:::benchmark_data_smithi001_156540_object47689:head v 158'11211, mlcod=158'11209)", 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.106743+0000", 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0037663890000000002, 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041410989999999996, 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106743+0000", 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106743+0000", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106743+0000", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6400000000000002e-07 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106747+0000", 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0600000000000001e-06 2024-09-14T04:06:48.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106755+0000", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9019999999999999e-06 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.106760+0000", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9609999999999996e-06 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.107463+0000", 2024-09-14T04:06:48.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00070297100000000004 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.107487+0000", 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4150000000000001e-05 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179285 9.f e158/156 9:f2c3e791:::benchmark_data_smithi001_156540_object47691:head v 158'11243, mlcod=158'11241)", 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.108849+0000", 2024-09-14T04:06:48.144 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0016597280000000001, 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020857900000000001, 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108849+0000", 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108849+0000", 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108851+0000", 2024-09-14T04:06:48.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.155e-06 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108857+0000", 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4110000000000001e-06 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108868+0000", 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1413000000000001e-05 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108875+0000", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7299999999999999e-06 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108946+0000", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1365000000000005e-05 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.108964+0000", 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7458000000000001e-05 2024-09-14T04:06:48.148 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.44714.0:179266 9.0 e158/135)", 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.110790+0000", 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4294967295.9997191, 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000197772, 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.149 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110790+0000", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110790+0000", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110791+0000", 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1739999999999999e-06 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110797+0000", 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0100000000000001e-06 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110808+0000", 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1270000000000001e-05 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.151 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110815+0000", 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2609999999999996e-06 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179287 9.d e158/135)", 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.110869+0000", 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4294967295.9996405, 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015871, 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.152 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110869+0000", 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110869+0000", 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.153 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110869+0000", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2e-07 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110874+0000", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5120000000000002e-06 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110882+0000", 2024-09-14T04:06:48.154 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9270000000000005e-06 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.110887+0000", 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1229999999999999e-06 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:179266 9.0 9:0075c919:::benchmark_data_smithi001_156540_object47672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.087386+0000", 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.023123492999999998, 2024-09-14T04:06:48.155 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.023687657000000001, 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 179266 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087386+0000", 2024-09-14T04:06:48.156 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087386+0000", 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087387+0000", 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.091e-06 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.157 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087391+0000", 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7380000000000002e-06 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087405+0000", 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3556e-05 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:06:48.158 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.087413+0000", 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1839999999999999e-06 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.095171+0000", 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0077574150000000001 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.095297+0000", 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012672000000000001 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.159 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.095417+0000", 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119566 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.098632+0000", 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032151800000000002 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:06:48.160 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:06:48.161 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 11 2024-09-14T04:06:48.161 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:06:48.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:48.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:48.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:48.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:48.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:48.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:48.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:48.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:48 smithi001 ceph-mon[112165]: pgmap v555: 89 pgs: 9 remapped+peering, 18 peering, 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 40 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 15760/254277 objects degraded (6.198%); 5380/254277 objects misplaced (2.116%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:48 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:48 smithi001 ceph-mon[107399]: pgmap v555: 89 pgs: 9 remapped+peering, 18 peering, 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 40 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 15760/254277 objects degraded (6.198%); 5380/254277 objects misplaced (2.116%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:48 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:48.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:48.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:48 smithi012 ceph-mon[101180]: pgmap v555: 89 pgs: 9 remapped+peering, 18 peering, 1 active+clean+remapped, 11 active+undersized, 10 active+undersized+degraded, 40 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 15760/254277 objects degraded (6.198%); 5380/254277 objects misplaced (2.116%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:48 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:48.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:48.887 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:48.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:48.988 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:49.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:49.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:49.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:49.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:49.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:49.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:06:49.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:49.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:49.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:49.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:49.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:49.701 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 15760/254277 objects degraded (6.198%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:49.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 15760/254277 objects degraded (6.198%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:49.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:49.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 15760/254277 objects degraded (6.198%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:49.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:49.904 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:49.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:50.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:50.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:50.103 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-09-14T04:06:50.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:50.178 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:50.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:50.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:50.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:50.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:50.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:50.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:50.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:50.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:50.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:50.802 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:50.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:50.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:50.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:50.932 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:06:50.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:51.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:51.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:51.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[112165]: pgmap v556: 89 pgs: 9 remapped+peering, 18 peering, 11 active+undersized, 10 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 15760/253224 objects degraded (6.224%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2166658827' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[107399]: pgmap v556: 89 pgs: 9 remapped+peering, 18 peering, 11 active+undersized, 10 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 15760/253224 objects degraded (6.224%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:51.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:51.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2166658827' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:50 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:51.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:51.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:50 smithi012 ceph-mon[101180]: pgmap v556: 89 pgs: 9 remapped+peering, 18 peering, 11 active+undersized, 10 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 15760/253224 objects degraded (6.224%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:06:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:06:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2166658827' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:50 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:06:51.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:51.454 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:51.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:51.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:51.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:51.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:51.733 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:51.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:51.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:51.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:51.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:51.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:52.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:52.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:52.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[107399]: osdmap e159: 8 total, 7 up, 8 in 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:51 smithi001 ceph-mon[112165]: osdmap e159: 8 total, 7 up, 8 in 2024-09-14T04:06:52.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:52.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:51 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:06:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:51 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:06:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:51 smithi012 ceph-mon[101180]: osdmap e159: 8 total, 7 up, 8 in 2024-09-14T04:06:52.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:52.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:52.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:52.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:52.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:52.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:52.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:52.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:52.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:52.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:52.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:52.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:53.029 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[107399]: pgmap v558: 89 pgs: 11 active+clean+remapped, 1 remapped+peering, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 3.9 MiB/s, 64 objects/s recovering 2024-09-14T04:06:53.029 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-14T04:06:53.029 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4291003752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:53.029 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[112165]: pgmap v558: 89 pgs: 11 active+clean+remapped, 1 remapped+peering, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 3.9 MiB/s, 64 objects/s recovering 2024-09-14T04:06:53.029 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-14T04:06:53.030 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4291003752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:53.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:53.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:53.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:53.208 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:06:53.208 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:06:53.208 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:06:53.208 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:06:53.209 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:06:53.210 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:06:53.210 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:06:53.210 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:06:53.210 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:06:53.210 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:06:53.233 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:53.265 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:52 smithi012 ceph-mon[101180]: pgmap v558: 89 pgs: 11 active+clean+remapped, 1 remapped+peering, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 3.9 MiB/s, 64 objects/s recovering 2024-09-14T04:06:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:52 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-14T04:06:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4291003752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:06:53.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:53.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:53.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:53.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:53.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:53.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:53.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:53.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:53.806 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:53.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:53.954 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:53.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:53.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:54.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:54.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:54.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:54.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:54.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:54.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:54.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:54.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:54.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:54.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:54.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:54.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:54.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:54.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:54.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:54.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:54.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:55.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.072 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:55.105 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:06:55.105 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:06:55.105 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:06:55.105 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2024-09-14T04:06:55.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-14T04:06:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:06:55.168 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:06:55.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:55.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:55.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:54 smithi001 ceph-mon[107399]: pgmap v559: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 12 B/s, 0 objects/s recovering 2024-09-14T04:06:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 20281/243579 objects degraded (8.326%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:54 smithi001 ceph-mon[112165]: pgmap v559: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 12 B/s, 0 objects/s recovering 2024-09-14T04:06:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 20281/243579 objects degraded (8.326%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:55.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:54 smithi012 ceph-mon[101180]: pgmap v559: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 20281/243579 objects degraded (8.326%); 2/243579 objects misplaced (0.001%); 12 B/s, 0 objects/s recovering 2024-09-14T04:06:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 20281/243579 objects degraded (8.326%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:55.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:55.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:55.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:55.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:55.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:55.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:55.762 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:06:55.762 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:06:55.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:55.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:55.872 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:55.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:55.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:55.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:06:55.990 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:56.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.059 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:56.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:56.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:56.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:56.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:56.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:56.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:56.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:56.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:56.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:06:56.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:56.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:06:56.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:06:56.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:56.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:56.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:56.876 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:56.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:56.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:56.927 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:06:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:06:56] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.51.0" 2024-09-14T04:06:56.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.022 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:57.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:57.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:57.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:57.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:06:57.201 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:56 smithi001 ceph-mon[107399]: pgmap v560: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 19966/240795 objects degraded (8.292%); 2/240795 objects misplaced (0.001%); 7.7 KiB/s, 1 objects/s recovering 2024-09-14T04:06:57.201 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:56 smithi001 ceph-mon[112165]: pgmap v560: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 19966/240795 objects degraded (8.292%); 2/240795 objects misplaced (0.001%); 7.7 KiB/s, 1 objects/s recovering 2024-09-14T04:06:57.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:57.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:56 smithi012 ceph-mon[101180]: pgmap v560: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 19966/240795 objects degraded (8.292%); 2/240795 objects misplaced (0.001%); 7.7 KiB/s, 1 objects/s recovering 2024-09-14T04:06:57.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:06:57.436 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:06:57.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.575 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:57.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:57.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:57.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:57.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:57.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:57.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:57.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:58.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:58.041 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:58.046 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:58.068 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:58.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:58.139 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:58.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:58.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:58.382 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:58.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:58.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:58.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:58.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:58.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:58.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:58.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:58.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:58.842 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:58.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:58.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:58.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:59.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:59.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:06:59.218 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:58 smithi001 ceph-mon[112165]: pgmap v561: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 18973/228087 objects degraded (8.318%); 2/228087 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:06:59.218 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:58 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 18973/228087 objects degraded (8.318%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:59.218 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:58 smithi001 ceph-mon[107399]: pgmap v561: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 18973/228087 objects degraded (8.318%); 2/228087 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:06:59.218 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:58 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 18973/228087 objects degraded (8.318%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:59.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:06:59.262 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:06:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:58 smithi012 ceph-mon[101180]: pgmap v561: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 18973/228087 objects degraded (8.318%); 2/228087 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:06:59.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:58 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 18973/228087 objects degraded (8.318%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:59.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:06:59.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:06:59.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:59.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:59.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:06:59.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:06:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:06:59.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:06:59.816 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:06:59.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:06:59.943 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-14T04:06:59.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:06:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:59.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:00.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:00.104 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:00.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:00.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:06:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:07:00.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:06:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:07:00.246 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:00.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:06:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:07:00.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:00.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:00.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:00.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:00.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:00.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:00.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:00.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:00.898 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:00.936 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:00.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:01.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:01.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:01.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:01.206 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[107399]: pgmap v562: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 18973/226116 objects degraded (8.391%); 2/226116 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:07:01.206 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:07:01.206 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[107399]: osdmap e160: 8 total, 7 up, 8 in 2024-09-14T04:07:01.207 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[112165]: pgmap v562: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 18973/226116 objects degraded (8.391%); 2/226116 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:07:01.207 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:07:01.207 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:00 smithi001 ceph-mon[112165]: osdmap e160: 8 total, 7 up, 8 in 2024-09-14T04:07:01.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:01.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:00 smithi012 ceph-mon[101180]: pgmap v562: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 18973/226116 objects degraded (8.391%); 2/226116 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2024-09-14T04:07:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/51193204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:07:01.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:00 smithi012 ceph-mon[101180]: osdmap e160: 8 total, 7 up, 8 in 2024-09-14T04:07:01.351 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:01.449 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:07:01.450 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:07:01.453 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:01.474 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:07:01.474 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:07:01.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:01.505 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:07:01.505 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:07:01.535 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:07:01.535 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:07:01.564 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:07:01.564 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:07:01.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.628 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:01.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:01.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:01.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:01.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:01.891 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:01.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:01.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:02.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:02.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:02.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1657758882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1657758882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:02.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1657758882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:02.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:02.471 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:02.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:02.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:02.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:02.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:02.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:02.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:02.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:02.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:02.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:03.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:03.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:03.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:03.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:03.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:02 smithi001 ceph-mon[107399]: pgmap v564: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:03.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:02 smithi001 ceph-mon[112165]: pgmap v564: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:03.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:03.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:02 smithi012 ceph-mon[101180]: pgmap v564: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:03.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:03.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:03.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:03.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:03.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:03.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:03.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:03.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:03.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:03.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:03.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:04.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:04.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:04.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:03 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 17675/212673 objects degraded (8.311%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:04.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:03 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 17675/212673 objects degraded (8.311%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:04.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:04.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:07:04.340 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:07:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:03 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 17675/212673 objects degraded (8.311%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:04.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.405 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:04.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:04.506 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:04.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:04.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:04.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:04.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:04.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:04.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:04.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:04.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:04.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:04.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:05.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:05.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:05.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:05.194 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[112165]: pgmap v565: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:05.194 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:05.194 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:05.195 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[107399]: pgmap v565: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:05.195 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:05.195 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:05.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:05.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:04 smithi012 ceph-mon[101180]: pgmap v565: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17675/212673 objects degraded (8.311%); 2/212673 objects misplaced (0.001%) 2024-09-14T04:07:05.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:05.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:05.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:05.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:05.473 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:05.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:05.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:05.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:05.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:05.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:05.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:05.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:06.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:06.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:06.186 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:06.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:06.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:06.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:07:06.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:06.386 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89043 8.a 8.216a668a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 551.75018219200001, 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.002694494, 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.387 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89043 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632793+0000", 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632795+0000", 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.99e-06 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632924+0000", 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012871200000000001 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632937+0000", 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2955999999999999e-05 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.632948+0000", 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1657999999999999e-05 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635088+0000", 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002139312 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635096+0000", 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5199999999999997e-06 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658069+0000", 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022972731999999999 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.658078+0000", 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7579999999999998e-06 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635476+0000", 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97739821800000004 2024-09-14T04:07:06.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635488+0000", 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1637999999999999e-05 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89049 8.3 8.20f81863 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 551.74796932799995, 2024-09-14T04:07:06.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000897207, 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89049 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:07:06.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635006+0000", 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635007+0000", 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.342e-06 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635067+0000", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0242999999999999e-05 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635079+0000", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1970999999999999e-05 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635088+0000", 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9269999999999994e-06 2024-09-14T04:07:06.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.650192+0000", 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015103836000000001 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.650202+0000", 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.679e-06 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635892+0000", 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98569037199999998 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635903+0000", 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0837e-05 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89050 8.3 8.612cea13 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:07:06.398 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 551.74786775899997, 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0008373749999999, 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89050 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:07:06.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635107+0000", 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635108+0000", 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3499999999999996e-07 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635163+0000", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4635000000000001e-05 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635172+0000", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6630000000000005e-06 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635180+0000", 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5320000000000001e-06 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653159+0000", 2024-09-14T04:07:06.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017979311000000001 2024-09-14T04:07:06.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653172+0000", 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2466999999999999e-05 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635935+0000", 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98276317899999999 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635945+0000", 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9529999999999994e-06 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89058 8.3 8.951b07a3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 551.74778537899999, 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000785891, 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89058 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.405 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635190+0000", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8599999999999998e-07 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635240+0000", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9608999999999999e-05 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635248+0000", 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1710000000000002e-06 2024-09-14T04:07:06.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635254+0000", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0059999999999998e-06 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653236+0000", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017982206000000001 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653245+0000", 2024-09-14T04:07:06.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4889999999999995e-06 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635969+0000", 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98272394100000005 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.635976+0000", 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.883e-06 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.409 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:89061 8.3 8.ab84b7b3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e108)", 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 551.74754971000004, 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.000835736, 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.410 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 89061 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635426+0000", 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0399999999999996e-07 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635477+0000", 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0652999999999998e-05 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635484+0000", 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5020000000000003e-06 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.635489+0000", 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2129999999999999e-06 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653317+0000", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017827393 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:54.653325+0000", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8499999999999994e-06 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.636247+0000", 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98292271799999997 2024-09-14T04:07:06.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:57:55.636261+0000", 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3903000000000001e-05 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:121341 8.3 8:c72b1ef3:::benchmark_data_smithi001_125658_object121340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 514.12779978399999, 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.014175067, 2024-09-14T04:07:06.415 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 121341 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255175+0000", 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255177+0000", 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.012e-06 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255204+0000", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6996999999999999e-05 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255218+0000", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3212e-05 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.255228+0000", 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9399999999999997e-06 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.259603+0000", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043758759999999999 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.259626+0000", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.211e-05 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.268869+0000", 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0092438880000000005 2024-09-14T04:07:06.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:32.268879+0000", 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7140000000000005e-06 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257381+0000", 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98850154899999998 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257528+0000", 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014704900000000001 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.257698+0000", 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000169789 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.266660+0000", 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089624390000000009 2024-09-14T04:07:06.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269181+0000", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002520834 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269280+0000", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9536000000000004e-05 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269321+0000", 2024-09-14T04:07:06.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0234999999999997e-05 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:58:33.269351+0000", 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9887e-05 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144668 8.a 8:53b134be:::benchmark_data_smithi001_125658_object19429:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:07:06.423 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 483.20368857199998, 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0089713979999999, 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144668 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:07:06.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179287+0000", 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179289+0000", 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8920000000000001e-06 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179295+0000", 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4099999999999996e-06 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179427+0000", 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013198600000000001 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179438+0000", 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1477999999999999e-05 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179923+0000", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048479699999999997 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.179935+0000", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1929999999999999e-05 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188185+0000", 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0082499250000001 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188258+0000", 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2979999999999996e-05 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144680 8.a 8:5ac33e5e:::benchmark_data_smithi001_125658_object19441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 483.20199119199998, 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.024066712, 2024-09-14T04:07:06.428 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144680 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180984+0000", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5200000000000002e-07 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180987+0000", 2024-09-14T04:07:06.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1730000000000002e-06 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180993+0000", 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1990000000000003e-06 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.180998+0000", 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8840000000000002e-06 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.181072+0000", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4634000000000004e-05 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.181081+0000", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2109999999999998e-06 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188740+0000", 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.007659597 2024-09-14T04:07:06.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.188893+0000", 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015260800000000001 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.189452+0000", 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055930699999999999 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204842+0000", 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015390019 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204883+0000", 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0751999999999999e-05 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.204945+0000", 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1816000000000006e-05 2024-09-14T04:07:06.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205003+0000", 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8137000000000003e-05 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205051+0000", 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7923000000000002e-05 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144687 8.a 8:5927234f:::benchmark_data_smithi001_125658_object19448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 483.19436159999998, 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0239874259999999, 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.436 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144687 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188614+0000", 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188615+0000", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.747e-06 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188620+0000", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4150000000000001e-06 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188635+0000", 2024-09-14T04:07:06.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5534999999999999e-05 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188643+0000", 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2910000000000002e-06 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188870+0000", 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000227631 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.188946+0000", 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5879999999999999e-05 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.201935+0000", 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0129886370000001 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.202067+0000", 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013186900000000001 2024-09-14T04:07:06.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.202199+0000", 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013186900000000001 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212260+0000", 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010061614 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212431+0000", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017089000000000001 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212503+0000", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2368999999999998e-05 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212558+0000", 2024-09-14T04:07:06.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4379999999999998e-05 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212601+0000", 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3299e-05 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:144694 8.a 8:5329308c:::benchmark_data_smithi001_125658_object19455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 483.18497661200001, 2024-09-14T04:07:06.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0241956910000001, 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 144694 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:07:06.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.197999+0000", 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198000+0000", 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.813e-06 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198007+0000", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.421e-06 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198022+0000", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5515e-05 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.198030+0000", 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9820000000000008e-06 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.199399+0000", 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0013681640000000001 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:03.199424+0000", 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5190000000000001e-05 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205099+0000", 2024-09-14T04:07:06.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0056756440000001 2024-09-14T04:07:06.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205233+0000", 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000133165 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.205364+0000", 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131347 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.212626+0000", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0072621350000000003 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222029+0000", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0094028069999999991 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222130+0000", 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010163299999999999 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222166+0000", 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5932000000000001e-05 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:04.222194+0000", 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7943e-05 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172983 8.1 8:897d1811:::benchmark_data_smithi001_125658_object47744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 447.85509020000001, 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.003049342, 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172983 2024-09-14T04:07:06.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527885+0000", 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527887+0000", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5600000000000001e-06 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527891+0000", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1359999999999999e-06 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527907+0000", 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6665000000000001e-05 2024-09-14T04:07:06.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.527914+0000", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0960000000000003e-06 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532469+0000", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045540349999999997 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532478+0000", 2024-09-14T04:07:06.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5400000000000001e-06 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523129+0000", 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99065059200000005 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523243+0000", 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114265 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:07:06.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.523315+0000", 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2384000000000004e-05 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530751+0000", 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0074354770000000002 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530769+0000", 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8235000000000001e-05 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530820+0000", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1110999999999999e-05 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530888+0000", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7537000000000001e-05 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530934+0000", 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6709000000000003e-05 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.458 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172987 8.1 8:82557b84:::benchmark_data_smithi001_125658_object47748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 447.85220479999998, 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.007018996, 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.459 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172987 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530770+0000", 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530772+0000", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.739e-06 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530775+0000", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.191e-06 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530785+0000", 2024-09-14T04:07:06.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4669999999999993e-06 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.530866+0000", 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1584999999999999e-05 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532502+0000", 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001635818 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.532508+0000", 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.908e-06 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527152+0000", 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99464408800000004 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527289+0000", 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013704 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.527406+0000", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116469 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.530983+0000", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035769510000000001 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537637+0000", 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0066542190000000003 2024-09-14T04:07:06.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537686+0000", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8838999999999998e-05 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537741+0000", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5473000000000002e-05 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.537789+0000", 2024-09-14T04:07:06.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8208999999999999e-05 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:172997 8.1 8:86774ea3:::benchmark_data_smithi001_125658_object47758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 447.84099496300001, 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0038687230000001, 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.466 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172997 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541980+0000", 2024-09-14T04:07:06.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541981+0000", 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.221e-06 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.541986+0000", 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.464e-06 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542001+0000", 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4718e-05 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542008+0000", 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.559e-06 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542161+0000", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000152905 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.542187+0000", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5644e-05 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531053+0000", 2024-09-14T04:07:06.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98886591700000004 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531180+0000", 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012756700000000001 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.531305+0000", 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012458099999999999 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545639+0000", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014334219 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545657+0000", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7595999999999999e-05 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545736+0000", 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9217999999999999e-05 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545789+0000", 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3072000000000003e-05 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545849+0000", 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0041999999999999e-05 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.473 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:173000 8.1 8:87fd81a4:::benchmark_data_smithi001_125658_object47761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 447.83870930699999, 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.001832101, 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 173000 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544266+0000", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544267+0000", 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.063e-06 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544270+0000", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1200000000000002e-06 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544279+0000", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3049999999999998e-06 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544285+0000", 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5729999999999998e-06 2024-09-14T04:07:06.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544354+0000", 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9206000000000001e-05 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:38.544363+0000", 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1050000000000001e-06 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534232+0000", 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98986859599999999 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534331+0000", 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9159000000000004e-05 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.534400+0000", 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9410000000000001e-05 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545927+0000", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01152623 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545941+0000", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4076e-05 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.545982+0000", 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0939000000000002e-05 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.546040+0000", 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8448e-05 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T03:59:39.546098+0000", 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7871000000000002e-05 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249045 8.7 8:e36474f6:::benchmark_data_smithi001_125658_object123806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 354.03867374100003, 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0581002930000001, 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249045 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.481 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344303+0000", 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4959999999999999e-06 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344307+0000", 2024-09-14T04:07:06.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0400000000000003e-06 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344317+0000", 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5759999999999995e-06 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344324+0000", 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7819999999999994e-06 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344858+0000", 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00053357000000000003 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344868+0000", 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6439999999999995e-06 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376162+0000", 2024-09-14T04:07:06.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0312939839999999 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376249+0000", 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7023999999999999e-05 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376790+0000", 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054109600000000002 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387731+0000", 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010941683000000001 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402294+0000", 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014562354 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402334+0000", 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0154000000000002e-05 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402368+0000", 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.417e-05 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402402+0000", 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3720000000000002e-05 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249083 8.7 8:e6756ecc:::benchmark_data_smithi001_125658_object123844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:07:06.491 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 353.99703182299999, 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016715622, 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249083 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385944+0000", 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5899999999999996e-07 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385947+0000", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.349e-06 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385959+0000", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1462e-05 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385965+0000", 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1870000000000002e-06 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:06.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386615+0000", 2024-09-14T04:07:06.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00064992000000000001 2024-09-14T04:07:06.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386640+0000", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5106e-05 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387765+0000", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0011247430000001 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387888+0000", 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123702 2024-09-14T04:07:06.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387975+0000", 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6299999999999997e-05 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.394473+0000", 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064983289999999997 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402469+0000", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079956669999999997 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402562+0000", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3187000000000001e-05 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402616+0000", 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4555999999999997e-05 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402659+0000", 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2555000000000002e-05 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161424 9.4 9:2dc0caff:::benchmark_data_smithi001_156540_object29830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 44.298914474, 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.134812339, 2024-09-14T04:07:06.500 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161424 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084063+0000", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.86e-06 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084068+0000", 2024-09-14T04:07:06.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3700000000000003e-06 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084082+0000", 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.438e-05 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084090+0000", 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9039999999999992e-06 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.096994+0000", 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012904186999999999 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.097111+0000", 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011634200000000001 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.097242+0000", 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131612 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.107203+0000", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099605730000000003 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.116776+0000", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0095725740000000004 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.119677+0000", 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029018899999999999 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.138479+0000", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018802003000000001 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.138492+0000", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2500999999999999e-05 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.218791+0000", 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0802988039999999 2024-09-14T04:07:06.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.218873+0000", 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2338999999999999e-05 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161431 9.4 9:2924b977:::benchmark_data_smithi001_156540_object29837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 44.270399415999997, 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.120078919, 2024-09-14T04:07:06.507 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161431 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112578+0000", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.922e-06 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112594+0000", 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6348e-05 2024-09-14T04:07:06.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112615+0000", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0559000000000001e-05 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112626+0000", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1138000000000001e-05 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112820+0000", 2024-09-14T04:07:06.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000194589 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112965+0000", 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014421099999999999 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.113115+0000", 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015039900000000001 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.511 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.117389+0000", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042743069999999998 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139019+0000", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021629598 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139027+0000", 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3319999999999992e-06 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219140+0000", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.080112307 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219255+0000", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011523099999999999 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232578+0000", 2024-09-14T04:07:06.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013323583999999999 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232655+0000", 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6394000000000001e-05 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161439 9.4 9:21cf609a:::benchmark_data_smithi001_156540_object29845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 44.252515936999998, 2024-09-14T04:07:06.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1061727830000001, 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161439 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130461+0000", 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5149999999999999e-06 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130465+0000", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3980000000000001e-06 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130477+0000", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2037e-05 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130487+0000", 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3989999999999993e-06 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139087+0000", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0086001710000000002 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139093+0000", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5880000000000003e-06 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219287+0000", 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0801940729999999 2024-09-14T04:07:06.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219394+0000", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106968 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219524+0000", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000129252 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.228456+0000", 2024-09-14T04:07:06.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089321609999999992 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232699+0000", 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042436389999999996 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236453+0000", 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037530580000000001 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236516+0000", 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3195999999999996e-05 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236632+0000", 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116328 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178538 9.4 9:2947cf7b:::benchmark_data_smithi001_156540_object46944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:07:06.521 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.028785533000001, 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.014113509, 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178538 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:06.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354191+0000", 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.238e-06 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354196+0000", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5809999999999998e-06 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354210+0000", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4250000000000001e-05 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354217+0000", 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3880000000000002e-06 2024-09-14T04:07:06.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354266+0000", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8795000000000003e-05 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354275+0000", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8089999999999999e-06 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364693+0000", 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0104177759999999 2024-09-14T04:07:06.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364776+0000", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3070000000000003e-05 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364860+0000", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4460000000000001e-05 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.367445+0000", 2024-09-14T04:07:06.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025850030000000002 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.367905+0000", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00046023199999999998 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368227+0000", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032120800000000002 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368258+0000", 2024-09-14T04:07:06.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1029999999999999e-05 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368303+0000", 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5668999999999999e-05 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:06.528 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:07:06.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:06.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:06.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:06.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:06.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:06.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:06] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.51.0" 2024-09-14T04:07:06.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:07.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:07.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:07.160 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:06 smithi001 ceph-mon[107399]: pgmap v566: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17369/209892 objects degraded (8.275%); 2/209892 objects misplaced (0.001%) 2024-09-14T04:07:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:06 smithi001 ceph-mon[112165]: pgmap v566: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17369/209892 objects degraded (8.275%); 2/209892 objects misplaced (0.001%) 2024-09-14T04:07:07.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:07.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:06 smithi012 ceph-mon[101180]: pgmap v566: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 17369/209892 objects degraded (8.275%); 2/209892 objects misplaced (0.001%) 2024-09-14T04:07:07.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:07.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:07.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:07.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:07.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:07.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:07:07.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:07.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:07.867 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:07.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:07.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:08.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:08.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:08.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:08.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:08.147 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:08.150 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:08.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:08.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:08.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:08.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:08.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:08.556 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:08.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:08.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:08.795 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:08.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:08.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:08.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:08.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:08.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:08.981 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:07:09.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:09.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:09.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:09.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[112165]: pgmap v567: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 16434/197757 objects degraded (8.310%); 2/197757 objects misplaced (0.001%) 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4109352811' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 16434/197757 objects degraded (8.310%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[107399]: pgmap v567: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 16434/197757 objects degraded (8.310%); 2/197757 objects misplaced (0.001%) 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4109352811' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:08 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 16434/197757 objects degraded (8.310%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:09.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:09.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:08 smithi012 ceph-mon[101180]: pgmap v567: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 16434/197757 objects degraded (8.310%); 2/197757 objects misplaced (0.001%) 2024-09-14T04:07:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4109352811' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:08 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:07:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:08 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 16434/197757 objects degraded (8.310%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:09.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:09.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:09.511 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:09.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:09.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:09.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:09.773 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:07:09.773 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:07:09.773 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:07:09.774 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.496162 2024-09-14T04:07:09.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 0.49616188905253866 2024-09-14T04:07:09.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:09.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:09.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:09.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:10.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:10.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:10.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:10.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[112165]: osdmap e161: 8 total, 7 up, 8 in 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:07:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:09 smithi001 ceph-mon[107399]: osdmap e161: 8 total, 7 up, 8 in 2024-09-14T04:07:10.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:10.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:09 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:07:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:09 smithi012 ceph-mon[101180]: osdmap e161: 8 total, 7 up, 8 in 2024-09-14T04:07:10.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:10.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:10.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:10.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:10.718 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:10.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:10.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:10.888 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:10.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:10.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:11.029 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:11.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:11.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:11.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:11.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:11.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:11.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:10 smithi012 ceph-mon[101180]: pgmap v569: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 16434/196023 objects degraded (8.384%); 2/196023 objects misplaced (0.001%) 2024-09-14T04:07:11.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:11.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:11.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:10 smithi001 ceph-mon[107399]: pgmap v569: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 16434/196023 objects degraded (8.384%); 2/196023 objects misplaced (0.001%) 2024-09-14T04:07:11.449 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:10 smithi001 ceph-mon[112165]: pgmap v569: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 16434/196023 objects degraded (8.384%); 2/196023 objects misplaced (0.001%) 2024-09-14T04:07:11.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:11.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:11.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:11.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:11.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:11.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:11.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:11.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:11.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:12.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:12.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:12.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:12.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:12.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:12.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:12.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:12.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:12.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:12.698 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:12.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:12.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:12.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:12.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:12.980 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:12.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:12.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:13.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:13.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:13.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:13.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:13 smithi012 ceph-mon[101180]: pgmap v570: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:13.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:13.367 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:13 smithi001 ceph-mon[107399]: pgmap v570: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:13.368 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:13 smithi001 ceph-mon[112165]: pgmap v570: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:13.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:13.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:13.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:13.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:13.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:13.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:13.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:13.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:13.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:13.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:14.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.080 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:14.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:14.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:14.217 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:14.262 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:07:14.263 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 15191/183357 objects degraded (8.285%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:14.263 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 15191/183357 objects degraded (8.285%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:14.264 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:14.294 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:07:14.294 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:07:14.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:14.324 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:07:14.324 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:07:14.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 15191/183357 objects degraded (8.285%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:14.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:14.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:07:14.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:07:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:14.386 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:07:14.387 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:07:14.419 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:07:14.420 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:07:14.449 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:07:14.449 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:07:14.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:14.477 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:07:14.478 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:07:14.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:14.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:14.627 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:14.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:14.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:14.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:14.899 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:14.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:15.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:15.007 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:07:15.007 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:07:15.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:15.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:07:15.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:15.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:15 smithi001 ceph-mon[112165]: pgmap v571: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:15.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/729401899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:15 smithi001 ceph-mon[107399]: pgmap v571: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:15.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/729401899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:15.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:15.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:15 smithi012 ceph-mon[101180]: pgmap v571: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 15191/183357 objects degraded (8.285%); 2/183357 objects misplaced (0.001%) 2024-09-14T04:07:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/729401899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:15.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:15.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:15.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:15.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:15.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:15.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:15.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:15.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:15.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:15.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:15.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:16.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:16.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:07:16.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:16.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:16.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:16.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:16.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:16.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:16.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:16.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:16.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:16.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:07:16.728 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:07:16.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:16.840 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:16.942 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:16] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.51.0" 2024-09-14T04:07:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:16.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:16.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:17.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:17.118 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:17.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:17.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:17.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:17 smithi012 ceph-mon[101180]: pgmap v572: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14879/180498 objects degraded (8.243%); 2/180498 objects misplaced (0.001%) 2024-09-14T04:07:17.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:17.373 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:17 smithi001 ceph-mon[107399]: pgmap v572: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14879/180498 objects degraded (8.243%); 2/180498 objects misplaced (0.001%) 2024-09-14T04:07:17.373 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:17 smithi001 ceph-mon[112165]: pgmap v572: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 14879/180498 objects degraded (8.243%); 2/180498 objects misplaced (0.001%) 2024-09-14T04:07:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:17.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:17.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:17.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:17.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:17.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:17.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:17.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:17.827 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:17.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:17.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:17.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:17.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:18.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:18.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:18.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:18.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:18.307 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:18.317 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:18.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:18.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:18.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:18.498 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:18.600 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:18.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:18.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:07:18.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:18.818 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:18.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:18.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:18.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:19.028 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.3 primary-affinity to 0.496162 (8325162) 2024-09-14T04:07:19.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:19.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:19.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:19.220 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[107399]: pgmap v573: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/167988 objects degraded (8.271%); 2/167988 objects misplaced (0.001%) 2024-09-14T04:07:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4208426224' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 13895/167988 objects degraded (8.271%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:19.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[112165]: pgmap v573: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/167988 objects degraded (8.271%); 2/167988 objects misplaced (0.001%) 2024-09-14T04:07:19.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4208426224' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 13895/167988 objects degraded (8.271%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:19.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:19 smithi012 ceph-mon[101180]: pgmap v573: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/167988 objects degraded (8.271%); 2/167988 objects misplaced (0.001%) 2024-09-14T04:07:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4208426224' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866} : dispatch 2024-09-14T04:07:19.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 13895/167988 objects degraded (8.271%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:19.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:19.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:19.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:19.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:19.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:19.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:19.726 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:19.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:19.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:19.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:19.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:19.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:20.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:20.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.124 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:20.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:20.226 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:20.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:20 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866}]': finished 2024-09-14T04:07:20.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:20 smithi012 ceph-mon[101180]: osdmap e162: 8 total, 7 up, 8 in 2024-09-14T04:07:20.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:20.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:20 smithi012 ceph-mon[101180]: osdmap e163: 8 total, 7 up, 8 in 2024-09-14T04:07:20.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:20.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866}]': finished 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[107399]: osdmap e162: 8 total, 7 up, 8 in 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[107399]: osdmap e163: 8 total, 7 up, 8 in 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.49616188905253866}]': finished 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[112165]: osdmap e162: 8 total, 7 up, 8 in 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:20 smithi001 ceph-mon[112165]: osdmap e163: 8 total, 7 up, 8 in 2024-09-14T04:07:20.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.545 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:20.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:20.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:20.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:20.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:20.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:20.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:20.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:20.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:20.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:20.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:21.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:21.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:21.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:21.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:21 smithi012 ceph-mon[101180]: pgmap v575: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/166278 objects degraded (8.356%); 2/166278 objects misplaced (0.001%) 2024-09-14T04:07:21.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:21 smithi012 ceph-mon[101180]: osdmap e164: 8 total, 7 up, 8 in 2024-09-14T04:07:21.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:21.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:21.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:21 smithi001 ceph-mon[107399]: pgmap v575: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/166278 objects degraded (8.356%); 2/166278 objects misplaced (0.001%) 2024-09-14T04:07:21.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:21 smithi001 ceph-mon[107399]: osdmap e164: 8 total, 7 up, 8 in 2024-09-14T04:07:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:21 smithi001 ceph-mon[112165]: pgmap v575: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 13895/166278 objects degraded (8.356%); 2/166278 objects misplaced (0.001%) 2024-09-14T04:07:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:21 smithi001 ceph-mon[112165]: osdmap e164: 8 total, 7 up, 8 in 2024-09-14T04:07:21.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:21.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:21.645 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:21.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:21.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:21.814 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:21.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:21.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:21.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:22.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:22.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:22.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:22.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:22.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:22.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:22.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:22.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:22.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:22.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:22.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:22.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:22.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:22.885 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:22.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:22.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.026 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:23.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:23.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:23.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:23.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:23 smithi001 ceph-mon[112165]: pgmap v578: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.33k op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:23.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:23 smithi001 ceph-mon[107399]: pgmap v578: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.33k op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:23.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:23.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:23.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:23.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:23 smithi012 ceph-mon[101180]: pgmap v578: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.33k op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:23.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:23.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:23.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.583 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:23.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:23.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:23.722 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:23.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:23.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:23.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:23.949 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:07:23.950 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:07:23.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:23.990 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:23.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:24.061 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:07:24.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:24.127 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:24.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:24.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:24.324 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:24.324 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 13014/156489 objects degraded (8.316%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:24.324 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:24.324 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 13014/156489 objects degraded (8.316%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:24.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 13014/156489 objects degraded (8.316%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:24.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:24.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:24.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:24.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:24.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:24.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:24.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:24.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:07:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:24.786 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.825 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:24.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:24.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:24.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:24.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:25.065 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:07:25.066 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:07:25.066 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T04:07:25.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:07:25.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:25.128 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:25.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:25.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:25.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:25 smithi012 ceph-mon[101180]: pgmap v579: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:25.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:25 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:07:25.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:25.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:25 smithi012 ceph-mon[101180]: osdmap e165: 8 total, 7 up, 8 in 2024-09-14T04:07:25.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:25.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:25.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[107399]: pgmap v579: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:25.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:07:25.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:25.419 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[107399]: osdmap e165: 8 total, 7 up, 8 in 2024-09-14T04:07:25.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[112165]: pgmap v579: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 13014/156489 objects degraded (8.316%); 2/156489 objects misplaced (0.001%); 0 B/s, 0 objects/s recovering 2024-09-14T04:07:25.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:07:25.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3771735367' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:25.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:25 smithi001 ceph-mon[112165]: osdmap e165: 8 total, 7 up, 8 in 2024-09-14T04:07:25.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:25.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:25.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:25.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:25.777 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:25.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:25.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:25.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:25.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:25.960 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:26.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:26.089 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:26.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:26.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:26.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:26.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:26.513 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:26.569 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:07:26.597 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:07:26.597 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:07:26.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:26.633 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:07:26.633 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:07:26.651 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:26.662 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:07:26.662 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:07:26.688 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:07:26.688 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:07:26.718 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:07:26.718 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:07:26.751 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:07:26.751 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:07:26.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:26.782 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:07:26.783 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:07:26.783 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:26] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.51.0" 2024-09-14T04:07:26.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:26.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:26.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:26.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:27.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:27.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:27.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:27.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:27.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:27.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:27 smithi012 ceph-mon[101180]: pgmap v581: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12698/151926 objects degraded (8.358%); 2/151926 objects misplaced (0.001%) 2024-09-14T04:07:27.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3390855609' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:27.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:27.427 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:27.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:27 smithi001 ceph-mon[107399]: pgmap v581: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12698/151926 objects degraded (8.358%); 2/151926 objects misplaced (0.001%) 2024-09-14T04:07:27.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3390855609' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:27 smithi001 ceph-mon[112165]: pgmap v581: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12698/151926 objects degraded (8.358%); 2/151926 objects misplaced (0.001%) 2024-09-14T04:07:27.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3390855609' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:27.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:27.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:27.654 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:27.756 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:27.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:27.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:27.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:07:27.944 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:27.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:28.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:07:28.049 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:07:28.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:28.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:28.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:28.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:28.309 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:28.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:28.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:28.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:28.478 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:28.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:28.613 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:28.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:28.748 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:28.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:28.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:28.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:28.988 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:29.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:29.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:29.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:29.255 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:29.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:29.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:29 smithi012 ceph-mon[101180]: pgmap v582: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 11791/142017 objects degraded (8.303%); 2/142017 objects misplaced (0.001%) 2024-09-14T04:07:29.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11791/142017 objects degraded (8.303%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:29.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:29.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:29.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:29 smithi001 ceph-mon[107399]: pgmap v582: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 11791/142017 objects degraded (8.303%); 2/142017 objects misplaced (0.001%) 2024-09-14T04:07:29.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11791/142017 objects degraded (8.303%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:29.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:29 smithi001 ceph-mon[112165]: pgmap v582: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 11791/142017 objects degraded (8.303%); 2/142017 objects misplaced (0.001%) 2024-09-14T04:07:29.468 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11791/142017 objects degraded (8.303%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:29.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:29.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:29.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:29.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:29.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:29.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:29.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:29.955 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:29.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:29.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:30.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:07:30.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:30.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:30.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:30.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.257 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:30.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:30.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.403 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:30.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:30.504 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:30.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:30.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:30.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:30.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:30.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:30.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:30.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:30.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:31.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:31.074 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:31.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:31.210 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:31.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:31.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:31.330 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:31 smithi001 ceph-mon[107399]: pgmap v583: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 11791/140118 objects degraded (8.415%); 2/140118 objects misplaced (0.001%) 2024-09-14T04:07:31.330 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:31 smithi001 ceph-mon[112165]: pgmap v583: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 11791/140118 objects degraded (8.415%); 2/140118 objects misplaced (0.001%) 2024-09-14T04:07:31.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:31 smithi012 ceph-mon[101180]: pgmap v583: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 11791/140118 objects degraded (8.415%); 2/140118 objects misplaced (0.001%) 2024-09-14T04:07:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.356 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:31.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:31.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:31.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:31.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:134818 9.6 e145/135 9:61808bc8:::benchmark_data_smithi001_156540_object3224:head v 145'8364, mlcod=145'8358)", 2024-09-14T04:07:31.630 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:05:49.205842+0000", 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 102.420076806, 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.029872573999999999, 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205842+0000", 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.631 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205842+0000", 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205843+0000", 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8700000000000005e-07 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205849+0000", 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7579999999999998e-06 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205860+0000", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1367e-05 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.205867+0000", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8299999999999998e-06 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.214594+0000", 2024-09-14T04:07:31.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087275149999999999 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.214652+0000", 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7624999999999998e-05 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.235397+0000", 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020744808 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:05:49.235715+0000", 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000317984 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:148216 9.6 e146/135 9:6cc7390f:::benchmark_data_smithi001_156540_object16622:head v 146'9210, mlcod=146'9207)", 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:03.940522+0000", 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 87.685397366000004, 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030581936000000001, 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.635 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940522+0000", 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940522+0000", 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940523+0000", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0079999999999999e-06 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940530+0000", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8420000000000005e-06 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940543+0000", 2024-09-14T04:07:31.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.255e-05 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940550+0000", 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2169999999999997e-06 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940946+0000", 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00039605 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.940964+0000", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7689e-05 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.970959+0000", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.029995279 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.971104+0000", 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000144301 2024-09-14T04:07:31.639 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150385 9.0 e146/135 9:0dda0f5c:::benchmark_data_smithi001_156540_object18791:head v 146'9282, mlcod=146'9280)", 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.778243+0000", 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.847676085000003, 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.083799586999999995, 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.640 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778243+0000", 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778243+0000", 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778244+0000", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6099999999999999e-07 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778251+0000", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2309999999999999e-06 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778262+0000", 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1388e-05 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778270+0000", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2830000000000004e-06 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778343+0000", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3046999999999994e-05 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.778362+0000", 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9051000000000001e-05 2024-09-14T04:07:31.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.861941+0000", 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.083579214999999998 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862042+0000", 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101411 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150397 9.2 e146/135 9:4c02ed60:::benchmark_data_smithi001_156540_object18803:head v 146'9295, mlcod=146'9293)", 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.785539+0000", 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.840380210000006, 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.086997283999999994, 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.785539+0000", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.785539+0000", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.785540+0000", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3549999999999999e-06 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.785548+0000", 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1179999999999992e-06 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862044+0000", 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.076496008000000004 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862057+0000", 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.254e-05 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862107+0000", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9929000000000003e-05 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862126+0000", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9318e-05 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.872440+0000", 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010314014 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.872536+0000", 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6001999999999993e-05 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150427 9.2 e146/135 9:477e1df5:::benchmark_data_smithi001_156540_object18833:head v 146'9299, mlcod=146'9293)", 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.862175+0000", 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.763744166999999, 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.029047595999999998, 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.649 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862175+0000", 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862175+0000", 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862175+0000", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.98e-07 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862178+0000", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0469999999999998e-06 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862183+0000", 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3210000000000004e-06 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862187+0000", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1339999999999997e-06 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.876468+0000", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014280169000000001 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.876487+0000", 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8876e-05 2024-09-14T04:07:31.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891182+0000", 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014695878000000001 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891222+0000", 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9972999999999998e-05 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150428 9.2 e146/135 9:44deb462:::benchmark_data_smithi001_156540_object18834:head v 146'9300, mlcod=146'9293)", 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.862193+0000", 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.763726032999998, 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.029063325000000001, 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.654 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862193+0000", 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862193+0000", 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.655 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862193+0000", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4700000000000001e-07 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862196+0000", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8289999999999999e-06 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862201+0000", 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4879999999999998e-06 2024-09-14T04:07:31.656 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862206+0000", 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6829999999999999e-06 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.880280+0000", 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018074283 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.880299+0000", 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8349e-05 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891229+0000", 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010930528 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891256+0000", 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7018000000000001e-05 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.658 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150437 9.2 e146/135 9:49f80fc4:::benchmark_data_smithi001_156540_object18843:head v 146'9301, mlcod=146'9293)", 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.862213+0000", 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.763706079000002, 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.029072866999999999, 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862213+0000", 2024-09-14T04:07:31.659 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.660 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862213+0000", 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.661 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862213+0000", 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0200000000000001e-07 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862220+0000", 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6880000000000002e-06 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862226+0000", 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9989999999999997e-06 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.862230+0000", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3980000000000001e-06 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.884127+0000", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021896395999999999 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.884146+0000", 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9786e-05 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891261+0000", 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0071144609999999999 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.891286+0000", 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4936999999999999e-05 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150560 9.2 e146/135 9:4267db74:::benchmark_data_smithi001_156540_object18966:head v 146'9310, mlcod=146'9308)", 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.980771+0000", 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.645148293000005, 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.037017347999999999, 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.980771+0000", 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.980771+0000", 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.980772+0000", 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4109999999999999e-06 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.982907+0000", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021351709999999999 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.982925+0000", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.789e-05 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.982934+0000", 2024-09-14T04:07:31.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8829999999999995e-06 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.982994+0000", 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0464000000000002e-05 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.983013+0000", 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8559e-05 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.017746+0000", 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.034733442000000003 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.017788+0000", 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1527999999999999e-05 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.669 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:150563 9.6 e146/135 9:6efe2549:::benchmark_data_smithi001_156540_object18969:head v 146'9372, mlcod=146'9370)", 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:06.984115+0000", 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 84.641804278999999, 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043646553999999997, 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.984115+0000", 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.984115+0000", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.984116+0000", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.71e-07 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:06.984122+0000", 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5010000000000001e-06 2024-09-14T04:07:31.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.011747+0000", 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.027624673999999998 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.011762+0000", 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4854999999999999e-05 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.011818+0000", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6573000000000002e-05 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.011848+0000", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9444000000000001e-05 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.027721+0000", 2024-09-14T04:07:31.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015873571 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:07.027761+0000", 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0064999999999997e-05 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:153856 9.2 e146/135 9:4de85f85:::benchmark_data_smithi001_156540_object22262:head v 146'9521, mlcod=146'9518)", 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:10.742961+0000", 2024-09-14T04:07:31.674 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 80.882957504000004, 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.028326833999999999, 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742961+0000", 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742961+0000", 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742962+0000", 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.23e-07 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742968+0000", 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4500000000000003e-06 2024-09-14T04:07:31.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742982+0000", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4762e-05 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.742989+0000", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.737e-06 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.752459+0000", 2024-09-14T04:07:31.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0094702290000000001 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.752480+0000", 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1061999999999999e-05 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.771250+0000", 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018769405999999999 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:10.771288+0000", 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8464999999999998e-05 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:162139 9.f e153/149 9:f73847b4:::benchmark_data_smithi001_156540_object30545:head v 153'10215, mlcod=153'10213)", 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:24.072722+0000", 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.553196657000001, 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075961192999999996, 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.679 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072722+0000", 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072722+0000", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072723+0000", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.09e-07 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072730+0000", 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5969999999999997e-06 2024-09-14T04:07:31.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072741+0000", 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0725e-05 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072747+0000", 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8859999999999998e-06 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072820+0000", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2137000000000004e-05 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072859+0000", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9419999999999999e-05 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.148647+0000", 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075788344999999993 2024-09-14T04:07:31.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.148683+0000", 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6174000000000003e-05 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:162134 9.2 e153/149 9:43b14d49:::benchmark_data_smithi001_156540_object30540:head v 153'9996, mlcod=153'9993)", 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:24.072763+0000", 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.553156397999999, 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075862983999999994, 2024-09-14T04:07:31.684 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072763+0000", 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072763+0000", 2024-09-14T04:07:31.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072763+0000", 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2600000000000001e-07 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072769+0000", 2024-09-14T04:07:31.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3009999999999997e-06 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072792+0000", 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3189000000000001e-05 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072799+0000", 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.576e-06 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072882+0000", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2336999999999995e-05 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.072903+0000", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0846999999999999e-05 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.148586+0000", 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075683505999999998 2024-09-14T04:07:31.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.148626+0000", 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9402e-05 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:162352 9.f e153/149 9:fa416cc0:::benchmark_data_smithi001_156540_object30758:head v 153'10236, mlcod=153'10233)", 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:24.435095+0000", 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.190823773000005, 2024-09-14T04:07:31.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.038304389000000001, 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435095+0000", 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435095+0000", 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435096+0000", 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999999e-07 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435104+0000", 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0150000000000003e-06 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435116+0000", 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1744e-05 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435123+0000", 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.464e-06 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435202+0000", 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8776999999999995e-05 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.435223+0000", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0792e-05 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.473365+0000", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.038141710000000002 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.473400+0000", 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5012000000000003e-05 2024-09-14T04:07:31.693 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:162365 9.7 e153/149 9:e38d06a6:::benchmark_data_smithi001_156540_object30771:head v 153'10119, mlcod=153'10117)", 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:24.439914+0000", 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.18600472, 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.033514752000000002, 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.694 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.439914+0000", 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.439914+0000", 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.439915+0000", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.85e-07 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.439921+0000", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2339999999999999e-06 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.464470+0000", 2024-09-14T04:07:31.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024548914000000002 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.464481+0000", 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1314999999999999e-05 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.464535+0000", 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3458999999999998e-05 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.464557+0000", 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2164999999999999e-05 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.473402+0000", 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088451970000000008 2024-09-14T04:07:31.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:24.473429+0000", 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6682999999999998e-05 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:166255 9.f e153/149 9:f68610a8:::benchmark_data_smithi001_156540_object34661:head v 153'10453, mlcod=153'10451)", 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:29.479787+0000", 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 62.146132238, 2024-09-14T04:07:31.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041006107, 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479787+0000", 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479787+0000", 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479787+0000", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4999999999999998e-07 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479791+0000", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0929999999999996e-06 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479797+0000", 2024-09-14T04:07:31.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2020000000000001e-06 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.479802+0000", 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.775e-06 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.489182+0000", 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0093798149999999997 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.489203+0000", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0829e-05 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.520271+0000", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031067829000000002 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.520793+0000", 2024-09-14T04:07:31.703 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00052221400000000001 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:166260 9.f e153/149 9:f3e75ef3:::benchmark_data_smithi001_156540_object34666:head v 153'10454, mlcod=153'10451)", 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:29.486240+0000", 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 62.139678918000001, 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.045805408999999998, 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.704 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486240+0000", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486240+0000", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486241+0000", 2024-09-14T04:07:31.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0869999999999999e-06 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486248+0000", 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.742e-06 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486260+0000", 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.204e-05 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.486268+0000", 2024-09-14T04:07:31.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7279999999999995e-06 2024-09-14T04:07:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:31.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:31.708 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.497869+0000", 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011601534 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.497933+0000", 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3351999999999999e-05 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.531822+0000", 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.033889137999999999 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.532045+0000", 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022378799999999999 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.710 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:166266 9.f e153/149 9:f125ec3d:::benchmark_data_smithi001_156540_object34672:head v 153'10455, mlcod=153'10451)", 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:29.498006+0000", 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 62.127913284000002, 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.034263203999999998, 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498006+0000", 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.711 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498006+0000", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498007+0000", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.048e-06 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498014+0000", 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9979999999999997e-06 2024-09-14T04:07:31.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498037+0000", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2875e-05 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.498045+0000", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.619e-06 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.501932+0000", 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038867860000000001 2024-09-14T04:07:31.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.502009+0000", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7286999999999994e-05 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.532049+0000", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030039349999999999 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.532269+0000", 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022024100000000001 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.714 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:166272 9.f e153/149 9:f99ecc26:::benchmark_data_smithi001_156540_object34678:head v 153'10456, mlcod=153'10451)", 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:29.503836+0000", 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 62.122083126, 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.028639159000000001, 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.715 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503836+0000", 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503836+0000", 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503837+0000", 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8100000000000001e-07 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503843+0000", 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2639999999999997e-06 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503857+0000", 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3533000000000001e-05 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.503865+0000", 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0499999999999992e-06 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.506397+0000", 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025321219999999999 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.506458+0000", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1121e-05 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.532272+0000", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.025813959000000001 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:29.532475+0000", 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000203229 2024-09-14T04:07:31.719 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:179274 9.2 e158/157 9:4beaf233:::benchmark_data_smithi001_156540_object47680:head v 158'11108, mlcod=158'11106)", 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.092526+0000", 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 43.533393289999999, 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030742496000000001, 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092526+0000", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092526+0000", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092527+0000", 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.09e-07 2024-09-14T04:07:31.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092533+0000", 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8290000000000002e-06 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092545+0000", 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1966e-05 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092561+0000", 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5147000000000001e-05 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.092630+0000", 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9329e-05 2024-09-14T04:07:31.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.097982+0000", 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053518790000000004 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.123215+0000", 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.025232841999999998 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.123268+0000", 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3594999999999998e-05 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:199304 9.6 e160/135 9:655aae58:::benchmark_data_smithi001_156540_object67710:head v 160'12343, mlcod=160'12339)", 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:07:08.023010+0000", 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 23.602908457000002, 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.027242972000000001, 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:07:31.725 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023010+0000", 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023010+0000", 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023012+0000", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.237e-06 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023019+0000", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.153e-06 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023032+0000", 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2889e-05 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023040+0000", 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7889999999999995e-06 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023275+0000", 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000235887 2024-09-14T04:07:31.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.023326+0000", 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0399000000000003e-05 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.050052+0000", 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.026725676 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:07:31.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:08.050253+0000", 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020194200000000001 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:07:31.730 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:07:31.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:31.849 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:31.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:31.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:31.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:32.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:32.131 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:32.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:32.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:32.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:32.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:32.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:32.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:32.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:32.641 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:32.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:32.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:32.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:32.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:32.939 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:32.966 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:32.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:33.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:33.133 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:33.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:07:33.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:33.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:33 smithi012 ceph-mon[101180]: pgmap v584: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:33.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:33.374 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:33 smithi001 ceph-mon[112165]: pgmap v584: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:33.374 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:33 smithi001 ceph-mon[107399]: pgmap v584: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.402 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:33.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:33.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:33.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:33.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:33.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:33.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:33.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:33.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:34.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:34.108 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:34.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.242 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:34.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:34.333 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10641/127569 objects degraded (8.341%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:34.334 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10641/127569 objects degraded (8.341%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:34.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:34.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10641/127569 objects degraded (8.341%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:34.380 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:34.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:34.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:34.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:34.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.652 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:34.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:34.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:34.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:34.897 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:34.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:34.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:35.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:35.045 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:07:35.046 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:07:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.080 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:35.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:35.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:35.220 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:35.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:35 smithi012 ceph-mon[101180]: pgmap v585: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:35.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:35.352 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:35.428 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[107399]: pgmap v585: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:35.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:35.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:35.429 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[112165]: pgmap v585: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 10641/127569 objects degraded (8.341%); 2/127569 objects misplaced (0.002%) 2024-09-14T04:07:35.429 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:35.429 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:35.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:35.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:35.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:35.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:35.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:35.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:35.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:35.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:35.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:35.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:35.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:36.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:36.059 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:36.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:36.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:36.258 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:36 smithi001 ceph-mon[112165]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:07:36.259 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":585,"stamp":"2024-09-14T04:07:33.704691+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2785613821,"num_objects":42523,"num_object_clones":0,"num_object_copies":127569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10641,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":42523,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":220966,"num_write_kb":8424274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10143,"num_bytes_recovered":664995604,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131765,"ondisk_log_size":131765,"up":233,"acting":245,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":284,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12999860,"kb_used_data":9350132,"kb_used_omap":28,"kb_used_meta":3649123,"kb_avail":736961356,"statfs":{"total":767960285184,"available":754648428544,"internally_reserved":0,"allocated":9574535168,"data_stored":9566466372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29302,"internal_metadata":3736702346},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,4,11,3,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-631767040,"num_objects":-9640,"num_object_clones":0,"num_object_copies":-28920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9640,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003337"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":118,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.064510+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:07:24.064510+0000","last_peered":"2024-09-14T04:07:24.064510+0000","last_clean":"2024-09-14T04:07:24.064510+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:07:24.064510+0000","last_undegraded":"2024-09-14T04:07:24.064510+0000","last_fullsized":"2024-09-14T04:07:24.064510+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":53,"reported_epoch":164,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:21.032622+0000","last_change":"2024-09-14T04:06:23.103542+0000","last_active":"2024-09-14T04:07:21.032622+0000","last_peered":"2024-09-14T04:07:21.032622+0000","last_clean":"2024-09-14T03:57:06.051481+0000","last_became_active":"2024-09-14T04:06:23.103542+0000","last_became_peered":"2024-09-14T04:06:23.103542+0000","last_unstale":"2024-09-14T04:07:21.032622+0000","last_undegraded":"2024-09-14T04:06:23.099648+0000","last_fullsized":"2024-09-14T04:06:23.099547+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:24.908981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.033137+0000","last_change":"2024-09-14T04:06:46.342718+0000","last_active":"2024-09-14T04:07:21.033137+0000","last_peered":"2024-09-14T04:07:21.033137+0000","last_clean":"2024-09-14T04:07:21.033137+0000","last_became_active":"2024-09-14T04:06:46.342351+0000","last_became_peered":"2024-09-14T04:06:46.342351+0000","last_unstale":"2024-09-14T04:07:21.033137+0000","last_undegraded":"2024-09-14T04:07:21.033137+0000","last_fullsized":"2024-09-14T04:07:21.033137+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:12:55.032175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":157,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.026967+0000","last_change":"2024-09-14T04:06:47.364842+0000","last_active":"2024-09-14T04:07:21.026967+0000","last_peered":"2024-09-14T04:07:21.026967+0000","last_clean":"2024-09-14T04:07:21.026967+0000","last_became_active":"2024-09-14T04:06:47.364474+0000","last_became_peered":"2024-09-14T04:06:47.364474+0000","last_unstale":"2024-09-14T04:07:21.026967+0000","last_undegraded":"2024-09-14T04:07:21.026967+0000","last_fullsized":"2024-09-14T04:07:21.026967+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:41.578211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":27,"reported_epoch":165,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:24.062456+0000","last_change":"2024-09-14T04:06:47.335797+0000","last_active":"2024-09-14T04:07:24.062456+0000","last_peered":"2024-09-14T04:07:24.062456+0000","last_clean":"2024-09-14T04:07:24.062456+0000","last_became_active":"2024-09-14T04:06:47.335313+0000","last_became_peered":"2024-09-14T04:06:47.335313+0000","last_unstale":"2024-09-14T04:07:24.062456+0000","last_undegraded":"2024-09-14T04:07:24.062456+0000","last_fullsized":"2024-09-14T04:07:24.062456+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:23.596405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,2],"avail_no_missing":["3","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.027172+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:07:21.027172+0000","last_peered":"2024-09-14T04:07:21.027172+0000","last_clean":"2024-09-14T04:07:21.027172+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:07:21.027172+0000","last_undegraded":"2024-09-14T04:07:21.027172+0000","last_fullsized":"2024-09-14T04:07:21.027172+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.062661+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:07:24.062661+0000","last_peered":"2024-09-14T04:07:24.062661+0000","last_clean":"2024-09-14T04:07:24.062661+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:07:24.062661+0000","last_undegraded":"2024-09-14T04:07:24.062661+0000","last_fullsized":"2024-09-14T04:07:24.062661+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":143,"reported_epoch":164,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:21.035852+0000","last_change":"2024-09-14T04:06:23.099768+0000","last_active":"2024-09-14T04:07:21.035852+0000","last_peered":"2024-09-14T04:07:21.035852+0000","last_clean":"2024-09-14T04:06:19.073434+0000","last_became_active":"2024-09-14T04:06:23.099768+0000","last_became_peered":"2024-09-14T04:06:23.099768+0000","last_unstale":"2024-09-14T04:07:21.035852+0000","last_undegraded":"2024-09-14T04:06:23.094987+0000","last_fullsized":"2024-09-14T04:06:23.094871+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:26:45.591014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"165'13834","reported_seq":13895,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:31.063551+0000","last_change":"2024-09-14T04:06:23.109890+0000","last_active":"2024-09-14T04:07:31.063551+0000","last_peered":"2024-09-14T04:07:31.063551+0000","last_clean":"2024-09-14T04:06:22.038432+0000","last_became_active":"2024-09-14T04:06:23.109890+0000","last_became_peered":"2024-09-14T04:06:23.109890+0000","last_unstale":"2024-09-14T04:07:31.063551+0000","last_undegraded":"2024-09-14T04:06:23.098815+0000","last_fullsized":"2024-09-14T04:06:23.098699+0000","mapping_epoch":152,"log_start":"142'4919","ondisk_log_start":"142'4919","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8915,"log_dups_size":0,"ondisk_log_size":8915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:28.877323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172490752,"num_objects":2632,"num_object_clones":0,"num_object_copies":7896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2632,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13834,"num_write_kb":526912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":2632}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":139,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:24.079347+0000","last_change":"2024-09-14T04:06:23.182126+0000","last_active":"2024-09-14T04:07:24.079347+0000","last_peered":"2024-09-14T04:07:24.079347+0000","last_clean":"2024-09-14T04:06:19.061254+0000","last_became_active":"2024-09-14T04:06:23.182126+0000","last_became_peered":"2024-09-14T04:06:23.182126+0000","last_unstale":"2024-09-14T04:07:24.079347+0000","last_undegraded":"2024-09-14T04:06:23.105973+0000","last_fullsized":"2024-09-14T04:06:23.105872+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:16:57.192093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.031669+0000","last_change":"2024-09-14T03:42:34.882833+0000","last_active":"2024-09-14T04:07:21.031669+0000","last_peered":"2024-09-14T04:07:21.031669+0000","last_clean":"2024-09-14T04:07:21.031669+0000","last_became_active":"2024-09-14T03:42:34.882521+0000","last_became_peered":"2024-09-14T03:42:34.882521+0000","last_unstale":"2024-09-14T04:07:21.031669+0000","last_undegraded":"2024-09-14T04:07:21.031669+0000","last_fullsized":"2024-09-14T04:07:21.031669+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:46.487465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"165'13967","reported_seq":14066,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:29.533228+0000","last_change":"2024-09-14T04:06:46.367680+0000","last_active":"2024-09-14T04:07:29.533228+0000","last_peered":"2024-09-14T04:07:29.533228+0000","last_clean":"2024-09-14T04:07:29.533228+0000","last_became_active":"2024-09-14T04:06:46.338906+0000","last_became_peered":"2024-09-14T04:06:46.338906+0000","last_unstale":"2024-09-14T04:07:29.533228+0000","last_undegraded":"2024-09-14T04:07:29.533228+0000","last_fullsized":"2024-09-14T04:07:29.533228+0000","mapping_epoch":156,"log_start":"143'7257","ondisk_log_start":"143'7257","created":135,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:03.824211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178061312,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13967,"num_write_kb":533888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":134,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.078154+0000","last_change":"2024-09-14T03:57:55.658263+0000","last_active":"2024-09-14T04:07:24.078154+0000","last_peered":"2024-09-14T04:07:24.078154+0000","last_clean":"2024-09-14T04:07:24.078154+0000","last_became_active":"2024-09-14T03:57:55.657817+0000","last_became_peered":"2024-09-14T03:57:55.657817+0000","last_unstale":"2024-09-14T04:07:24.078154+0000","last_undegraded":"2024-09-14T04:07:24.078154+0000","last_fullsized":"2024-09-14T04:07:24.078154+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:36.196909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.031203+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:07:21.031203+0000","last_peered":"2024-09-14T04:07:21.031203+0000","last_clean":"2024-09-14T04:07:21.031203+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:07:21.031203+0000","last_undegraded":"2024-09-14T04:07:21.031203+0000","last_fullsized":"2024-09-14T04:07:21.031203+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"165'13819","reported_seq":13885,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:31.053141+0000","last_change":"2024-09-14T04:06:46.427059+0000","last_active":"2024-09-14T04:07:31.053141+0000","last_peered":"2024-09-14T04:07:31.053141+0000","last_clean":"2024-09-14T04:07:31.053141+0000","last_became_active":"2024-09-14T04:06:46.372804+0000","last_became_peered":"2024-09-14T04:06:46.372804+0000","last_unstale":"2024-09-14T04:07:31.053141+0000","last_undegraded":"2024-09-14T04:07:31.053141+0000","last_fullsized":"2024-09-14T04:07:31.053141+0000","mapping_epoch":156,"log_start":"142'4930","ondisk_log_start":"142'4930","created":135,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:14:11.078335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13819,"num_write_kb":527040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":118,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.086595+0000","last_change":"2024-09-14T03:42:34.641869+0000","last_active":"2024-09-14T04:07:24.086595+0000","last_peered":"2024-09-14T04:07:24.086595+0000","last_clean":"2024-09-14T04:07:24.086595+0000","last_became_active":"2024-09-14T03:42:34.641596+0000","last_became_peered":"2024-09-14T03:42:34.641596+0000","last_unstale":"2024-09-14T04:07:24.086595+0000","last_undegraded":"2024-09-14T04:07:24.086595+0000","last_fullsized":"2024-09-14T04:07:24.086595+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:47.928806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":129,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.037353+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:07:21.037353+0000","last_peered":"2024-09-14T04:07:21.037353+0000","last_clean":"2024-09-14T04:07:21.037353+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:07:21.037353+0000","last_undegraded":"2024-09-14T04:07:21.037353+0000","last_fullsized":"2024-09-14T04:07:21.037353+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"165'13667","reported_seq":13707,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:31.021603+0000","last_change":"2024-09-14T04:04:00.899930+0000","last_active":"2024-09-14T04:07:31.021603+0000","last_peered":"2024-09-14T04:07:31.021603+0000","last_clean":"2024-09-14T04:07:31.021603+0000","last_became_active":"2024-09-14T04:04:00.899336+0000","last_became_peered":"2024-09-14T04:04:00.899336+0000","last_unstale":"2024-09-14T04:07:31.021603+0000","last_undegraded":"2024-09-14T04:07:31.021603+0000","last_fullsized":"2024-09-14T04:07:31.021603+0000","mapping_epoch":135,"log_start":"141'4779","ondisk_log_start":"141'4779","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8888,"log_dups_size":0,"ondisk_log_size":8888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:52.223239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13667,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":150,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.055040+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:07:21.055040+0000","last_peered":"2024-09-14T04:07:21.055040+0000","last_clean":"2024-09-14T04:07:21.055040+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:07:21.055040+0000","last_undegraded":"2024-09-14T04:07:21.055040+0000","last_fullsized":"2024-09-14T04:07:21.055040+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.068697+0000","last_change":"2024-09-14T04:06:23.116743+0000","last_active":"2024-09-14T04:07:24.068697+0000","last_peered":"2024-09-14T04:07:24.068697+0000","last_clean":"2024-09-14T03:57:08.970573+0000","last_became_active":"2024-09-14T04:06:23.116743+0000","last_became_peered":"2024-09-14T04:06:23.116743+0000","last_unstale":"2024-09-14T04:07:24.068697+0000","last_undegraded":"2024-09-14T04:07:24.068697+0000","last_fullsized":"2024-09-14T04:06:23.098970+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:28:44.120442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"165'13834","reported_seq":21258,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:30.553088+0000","last_change":"2024-09-14T04:07:20.137632+0000","last_active":"2024-09-14T04:07:30.553088+0000","last_peered":"2024-09-14T04:07:30.553088+0000","last_clean":"2024-09-14T04:07:30.553088+0000","last_became_active":"2024-09-14T04:07:20.054011+0000","last_became_peered":"2024-09-14T04:07:20.054011+0000","last_unstale":"2024-09-14T04:07:30.553088+0000","last_undegraded":"2024-09-14T04:07:30.553088+0000","last_fullsized":"2024-09-14T04:07:30.553088+0000","mapping_epoch":162,"log_start":"142'5451","ondisk_log_start":"142'5451","created":135,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8383,"log_dups_size":0,"ondisk_log_size":8383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:24:13.453302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13834,"num_write_kb":526720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3670,"num_bytes_recovered":240386048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":34,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.044376+0000","last_change":"2024-09-14T04:07:21.043755+0000","last_active":"2024-09-14T04:07:21.044376+0000","last_peered":"2024-09-14T04:07:21.044376+0000","last_clean":"2024-09-14T04:07:21.044376+0000","last_became_active":"2024-09-14T04:07:21.043326+0000","last_became_peered":"2024-09-14T04:07:21.043326+0000","last_unstale":"2024-09-14T04:07:21.044376+0000","last_undegraded":"2024-09-14T04:07:21.044376+0000","last_fullsized":"2024-09-14T04:07:21.044376+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:48:20.157218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":149,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.033587+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:07:21.033587+0000","last_peered":"2024-09-14T04:07:21.033587+0000","last_clean":"2024-09-14T04:07:21.033587+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:07:21.033587+0000","last_undegraded":"2024-09-14T04:07:21.033587+0000","last_fullsized":"2024-09-14T04:07:21.033587+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"165'13585","reported_seq":13657,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:29.556995+0000","last_change":"2024-09-14T04:04:00.899509+0000","last_active":"2024-09-14T04:07:29.556995+0000","last_peered":"2024-09-14T04:07:29.556995+0000","last_clean":"2024-09-14T04:07:29.556995+0000","last_became_active":"2024-09-14T04:04:00.899240+0000","last_became_peered":"2024-09-14T04:04:00.899240+0000","last_unstale":"2024-09-14T04:07:29.556995+0000","last_undegraded":"2024-09-14T04:07:29.556995+0000","last_fullsized":"2024-09-14T04:07:29.556995+0000","mapping_epoch":135,"log_start":"143'6856","ondisk_log_start":"143'6856","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:56.471259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13585,"num_write_kb":520192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":156,"reported_epoch":164,"state":"active+undersized","last_fresh":"2024-09-14T04:07:21.037498+0000","last_change":"2024-09-14T04:06:23.103636+0000","last_active":"2024-09-14T04:07:21.037498+0000","last_peered":"2024-09-14T04:07:21.037498+0000","last_clean":"2024-09-14T04:06:19.073398+0000","last_became_active":"2024-09-14T04:06:23.103636+0000","last_became_peered":"2024-09-14T04:06:23.103636+0000","last_unstale":"2024-09-14T04:07:21.037498+0000","last_undegraded":"2024-09-14T04:07:21.037498+0000","last_fullsized":"2024-09-14T04:06:23.094792+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:54.980439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":146,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.082153+0000","last_change":"2024-09-14T04:01:59.052174+0000","last_active":"2024-09-14T04:07:24.082153+0000","last_peered":"2024-09-14T04:07:24.082153+0000","last_clean":"2024-09-14T04:07:24.082153+0000","last_became_active":"2024-09-14T04:01:59.051831+0000","last_became_peered":"2024-09-14T04:01:59.051831+0000","last_unstale":"2024-09-14T04:07:24.082153+0000","last_undegraded":"2024-09-14T04:07:24.082153+0000","last_fullsized":"2024-09-14T04:07:24.082153+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:15.224478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"165'13790","reported_seq":18625,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:30.852619+0000","last_change":"2024-09-14T04:06:47.436350+0000","last_active":"2024-09-14T04:07:30.852619+0000","last_peered":"2024-09-14T04:07:30.852619+0000","last_clean":"2024-09-14T04:07:30.852619+0000","last_became_active":"2024-09-14T04:06:47.372976+0000","last_became_peered":"2024-09-14T04:06:47.372976+0000","last_unstale":"2024-09-14T04:07:30.852619+0000","last_undegraded":"2024-09-14T04:07:30.852619+0000","last_fullsized":"2024-09-14T04:07:30.852619+0000","mapping_epoch":157,"log_start":"142'6030","ondisk_log_start":"142'6030","created":135,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:14.235919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171966464,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13790,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2384,"num_bytes_recovered":156041216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.031422+0000","last_change":"2024-09-14T04:01:59.055460+0000","last_active":"2024-09-14T04:07:21.031422+0000","last_peered":"2024-09-14T04:07:21.031422+0000","last_clean":"2024-09-14T04:07:21.031422+0000","last_became_active":"2024-09-14T04:01:59.055064+0000","last_became_peered":"2024-09-14T04:01:59.055064+0000","last_unstale":"2024-09-14T04:07:21.031422+0000","last_undegraded":"2024-09-14T04:07:21.031422+0000","last_fullsized":"2024-09-14T04:07:21.031422+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:52.089107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.081864+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:07:24.081864+0000","last_peered":"2024-09-14T04:07:24.081864+0000","last_clean":"2024-09-14T04:07:24.081864+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:07:24.081864+0000","last_undegraded":"2024-09-14T04:07:24.081864+0000","last_fullsized":"2024-09-14T04:07:24.081864+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"165'13770","reported_seq":15890,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:31.521255+0000","last_change":"2024-09-14T04:06:47.364211+0000","last_active":"2024-09-14T04:07:31.521255+0000","last_peered":"2024-09-14T04:07:31.521255+0000","last_clean":"2024-09-14T04:07:31.521255+0000","last_became_active":"2024-09-14T04:06:47.332574+0000","last_became_peered":"2024-09-14T04:06:47.332574+0000","last_unstale":"2024-09-14T04:07:31.521255+0000","last_undegraded":"2024-09-14T04:07:31.521255+0000","last_fullsized":"2024-09-14T04:07:31.521255+0000","mapping_epoch":157,"log_start":"142'6026","ondisk_log_start":"142'6026","created":135,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:11.539293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13770,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1016,"num_bytes_recovered":66584576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.062483+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:07:24.062483+0000","last_peered":"2024-09-14T04:07:24.062483+0000","last_clean":"2024-09-14T04:07:24.062483+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:07:24.062483+0000","last_undegraded":"2024-09-14T04:07:24.062483+0000","last_fullsized":"2024-09-14T04:07:24.062483+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":164,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.069533+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:07:24.069533+0000","last_peered":"2024-09-14T04:07:24.069533+0000","last_clean":"2024-09-14T04:07:24.069533+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:07:24.069533+0000","last_undegraded":"2024-09-14T04:07:24.069533+0000","last_fullsized":"2024-09-14T04:07:24.069533+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"165'13886","reported_seq":17605,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:31.034659+0000","last_change":"2024-09-14T04:07:20.031541+0000","last_active":"2024-09-14T04:07:31.034659+0000","last_peered":"2024-09-14T04:07:31.034659+0000","last_clean":"2024-09-14T04:06:14.977356+0000","last_became_active":"2024-09-14T04:07:20.031541+0000","last_became_peered":"2024-09-14T04:07:20.031541+0000","last_unstale":"2024-09-14T04:07:31.034659+0000","last_undegraded":"2024-09-14T04:07:20.029034+0000","last_fullsized":"2024-09-14T04:07:20.028907+0000","mapping_epoch":162,"log_start":"142'5553","ondisk_log_start":"142'5553","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8333,"log_dups_size":0,"ondisk_log_size":8333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:04.224282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176029696,"num_objects":2686,"num_object_clones":0,"num_object_copies":8058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13886,"num_write_kb":530304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1820,"num_bytes_recovered":119275520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2686}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":469,"reported_epoch":164,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:21.027784+0000","last_change":"2024-09-14T04:06:23.108559+0000","last_active":"2024-09-14T04:07:21.027784+0000","last_peered":"2024-09-14T04:07:21.027784+0000","last_clean":"2024-09-14T04:06:19.062284+0000","last_became_active":"2024-09-14T04:06:23.108559+0000","last_became_peered":"2024-09-14T04:06:23.108559+0000","last_unstale":"2024-09-14T04:07:21.027784+0000","last_undegraded":"2024-09-14T04:06:23.102934+0000","last_fullsized":"2024-09-14T04:06:23.101010+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T03:38:53.062469+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:38.980216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.030070+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:07:21.030070+0000","last_peered":"2024-09-14T04:07:21.030070+0000","last_clean":"2024-09-14T04:07:21.030070+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:07:21.030070+0000","last_undegraded":"2024-09-14T04:07:21.030070+0000","last_fullsized":"2024-09-14T04:07:21.030070+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":93,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.034245+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:07:21.034245+0000","last_peered":"2024-09-14T04:07:21.034245+0000","last_clean":"2024-09-14T04:07:21.034245+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:07:21.034245+0000","last_undegraded":"2024-09-14T04:07:21.034245+0000","last_fullsized":"2024-09-14T04:07:21.034245+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.035842+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:07:21.035842+0000","last_peered":"2024-09-14T04:07:21.035842+0000","last_clean":"2024-09-14T04:07:21.035842+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:07:21.035842+0000","last_undegraded":"2024-09-14T04:07:21.035842+0000","last_fullsized":"2024-09-14T04:07:21.035842+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"165'13660","reported_seq":13723,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:30.530380+0000","last_change":"2024-09-14T04:06:23.099741+0000","last_active":"2024-09-14T04:07:30.530380+0000","last_peered":"2024-09-14T04:07:30.530380+0000","last_clean":"2024-09-14T04:06:22.070355+0000","last_became_active":"2024-09-14T04:06:23.099741+0000","last_became_peered":"2024-09-14T04:06:23.099741+0000","last_unstale":"2024-09-14T04:07:30.530380+0000","last_undegraded":"2024-09-14T04:06:23.095246+0000","last_fullsized":"2024-09-14T04:06:23.095131+0000","mapping_epoch":152,"log_start":"141'3913","ondisk_log_start":"141'3913","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":9747,"log_dups_size":0,"ondisk_log_size":9747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:14.475293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177471488,"num_objects":2708,"num_object_clones":0,"num_object_copies":8124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13660,"num_write_kb":523776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":2708}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":167,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.077297+0000","last_change":"2024-09-14T04:06:23.112663+0000","last_active":"2024-09-14T04:07:24.077297+0000","last_peered":"2024-09-14T04:07:24.077297+0000","last_clean":"2024-09-14T04:06:19.060723+0000","last_became_active":"2024-09-14T04:06:23.112663+0000","last_became_peered":"2024-09-14T04:06:23.112663+0000","last_unstale":"2024-09-14T04:07:24.077297+0000","last_undegraded":"2024-09-14T04:07:24.077297+0000","last_fullsized":"2024-09-14T04:06:23.101600+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:37.867989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":158,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.031096+0000","last_change":"2024-09-14T04:06:47.344591+0000","last_active":"2024-09-14T04:07:21.031096+0000","last_peered":"2024-09-14T04:07:21.031096+0000","last_clean":"2024-09-14T04:07:21.031096+0000","last_became_active":"2024-09-14T04:06:47.344133+0000","last_became_peered":"2024-09-14T04:06:47.344133+0000","last_unstale":"2024-09-14T04:07:21.031096+0000","last_undegraded":"2024-09-14T04:07:21.031096+0000","last_fullsized":"2024-09-14T04:07:21.031096+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:32:47.229087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,5],"avail_no_missing":["4","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":118,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.069726+0000","last_change":"2024-09-14T04:06:23.101343+0000","last_active":"2024-09-14T04:07:24.069726+0000","last_peered":"2024-09-14T04:07:24.069726+0000","last_clean":"2024-09-14T04:01:26.406289+0000","last_became_active":"2024-09-14T04:06:23.101343+0000","last_became_peered":"2024-09-14T04:06:23.101343+0000","last_unstale":"2024-09-14T04:07:24.069726+0000","last_undegraded":"2024-09-14T04:07:24.069726+0000","last_fullsized":"2024-09-14T04:06:23.098350+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:51.746338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"165'13626","reported_seq":13666,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:31.070808+0000","last_change":"2024-09-14T04:04:00.899703+0000","last_active":"2024-09-14T04:07:31.070808+0000","last_peered":"2024-09-14T04:07:31.070808+0000","last_clean":"2024-09-14T04:07:31.070808+0000","last_became_active":"2024-09-14T04:04:00.899248+0000","last_became_peered":"2024-09-14T04:04:00.899248+0000","last_unstale":"2024-09-14T04:07:31.070808+0000","last_undegraded":"2024-09-14T04:07:31.070808+0000","last_fullsized":"2024-09-14T04:07:31.070808+0000","mapping_epoch":135,"log_start":"141'4781","ondisk_log_start":"141'4781","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8845,"log_dups_size":0,"ondisk_log_size":8845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:40.087745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179044352,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13626,"num_write_kb":523456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":200,"reported_epoch":165,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:24.064085+0000","last_change":"2024-09-14T04:06:47.333122+0000","last_active":"2024-09-14T04:07:24.064085+0000","last_peered":"2024-09-14T04:07:24.064085+0000","last_clean":"2024-09-14T04:07:24.064085+0000","last_became_active":"2024-09-14T04:06:47.332683+0000","last_became_peered":"2024-09-14T04:06:47.332683+0000","last_unstale":"2024-09-14T04:07:24.064085+0000","last_undegraded":"2024-09-14T04:07:24.064085+0000","last_fullsized":"2024-09-14T04:07:24.064085+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:18:43.810483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,1],"avail_no_missing":["2","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.068966+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:07:24.068966+0000","last_peered":"2024-09-14T04:07:24.068966+0000","last_clean":"2024-09-14T04:07:24.068966+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:07:24.068966+0000","last_undegraded":"2024-09-14T04:07:24.068966+0000","last_fullsized":"2024-09-14T04:07:24.068966+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":197,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.069239+0000","last_change":"2024-09-14T04:06:23.104016+0000","last_active":"2024-09-14T04:07:24.069239+0000","last_peered":"2024-09-14T04:07:24.069239+0000","last_clean":"2024-09-14T04:06:19.062578+0000","last_became_active":"2024-09-14T04:06:23.104016+0000","last_became_peered":"2024-09-14T04:06:23.104016+0000","last_unstale":"2024-09-14T04:07:24.069239+0000","last_undegraded":"2024-09-14T04:07:24.069239+0000","last_fullsized":"2024-09-14T04:06:23.098838+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:14:24.709322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"165'13722","reported_seq":13785,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:30.811085+0000","last_change":"2024-09-14T04:06:23.107709+0000","last_active":"2024-09-14T04:07:30.811085+0000","last_peered":"2024-09-14T04:07:30.811085+0000","last_clean":"2024-09-14T04:06:22.004970+0000","last_became_active":"2024-09-14T04:06:23.107709+0000","last_became_peered":"2024-09-14T04:06:23.107709+0000","last_unstale":"2024-09-14T04:07:30.811085+0000","last_undegraded":"2024-09-14T04:06:23.102125+0000","last_fullsized":"2024-09-14T04:06:23.102014+0000","mapping_epoch":152,"log_start":"142'6027","ondisk_log_start":"142'6027","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":7695,"log_dups_size":0,"ondisk_log_size":7695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:54.392766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170524672,"num_objects":2602,"num_object_clones":0,"num_object_copies":7806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2602,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13722,"num_write_kb":522368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":2602}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":147,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.034330+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:07:21.034330+0000","last_peered":"2024-09-14T04:07:21.034330+0000","last_clean":"2024-09-14T04:07:21.034330+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:07:21.034330+0000","last_undegraded":"2024-09-14T04:07:21.034330+0000","last_fullsized":"2024-09-14T04:07:21.034330+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":138,"reported_epoch":164,"state":"active+undersized","last_fresh":"2024-09-14T04:07:21.032755+0000","last_change":"2024-09-14T04:06:23.178242+0000","last_active":"2024-09-14T04:07:21.032755+0000","last_peered":"2024-09-14T04:07:21.032755+0000","last_clean":"2024-09-14T04:06:19.068163+0000","last_became_active":"2024-09-14T04:06:23.178242+0000","last_became_peered":"2024-09-14T04:06:23.178242+0000","last_unstale":"2024-09-14T04:07:21.032755+0000","last_undegraded":"2024-09-14T04:07:21.032755+0000","last_fullsized":"2024-09-14T04:06:23.098620+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:31.713178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":161,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.030192+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:07:21.030192+0000","last_peered":"2024-09-14T04:07:21.030192+0000","last_clean":"2024-09-14T04:07:21.030192+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:07:21.030192+0000","last_undegraded":"2024-09-14T04:07:21.030192+0000","last_fullsized":"2024-09-14T04:07:21.030192+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"165'13833","reported_seq":15463,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:31.078854+0000","last_change":"2024-09-14T04:06:46.402104+0000","last_active":"2024-09-14T04:07:31.078854+0000","last_peered":"2024-09-14T04:07:31.078854+0000","last_clean":"2024-09-14T04:07:31.078854+0000","last_became_active":"2024-09-14T04:06:46.343888+0000","last_became_peered":"2024-09-14T04:06:46.343888+0000","last_unstale":"2024-09-14T04:07:31.078854+0000","last_undegraded":"2024-09-14T04:07:31.078854+0000","last_fullsized":"2024-09-14T04:07:31.078854+0000","mapping_epoch":156,"log_start":"142'4922","ondisk_log_start":"142'4922","created":135,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8911,"log_dups_size":0,"ondisk_log_size":8911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:00.328827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13833,"num_write_kb":528128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":51380224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.033338+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:07:21.033338+0000","last_peered":"2024-09-14T04:07:21.033338+0000","last_clean":"2024-09-14T04:07:21.033338+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:07:21.033338+0000","last_undegraded":"2024-09-14T04:07:21.033338+0000","last_fullsized":"2024-09-14T04:07:21.033338+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":211,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.068967+0000","last_change":"2024-09-14T04:06:23.110251+0000","last_active":"2024-09-14T04:07:24.068967+0000","last_peered":"2024-09-14T04:07:24.068967+0000","last_clean":"2024-09-14T04:06:19.063292+0000","last_became_active":"2024-09-14T04:06:23.110251+0000","last_became_peered":"2024-09-14T04:06:23.110251+0000","last_unstale":"2024-09-14T04:07:24.068967+0000","last_undegraded":"2024-09-14T04:07:24.068967+0000","last_fullsized":"2024-09-14T04:06:23.102133+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:00.916012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.069617+0000","last_change":"2024-09-14T03:57:55.667306+0000","last_active":"2024-09-14T04:07:24.069617+0000","last_peered":"2024-09-14T04:07:24.069617+0000","last_clean":"2024-09-14T04:07:24.069617+0000","last_became_active":"2024-09-14T03:57:55.661348+0000","last_became_peered":"2024-09-14T03:57:55.661348+0000","last_unstale":"2024-09-14T04:07:24.069617+0000","last_undegraded":"2024-09-14T04:07:24.069617+0000","last_fullsized":"2024-09-14T04:07:24.069617+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:25.408014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"165'13991","reported_seq":14030,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:30.856487+0000","last_change":"2024-09-14T04:04:00.932107+0000","last_active":"2024-09-14T04:07:30.856487+0000","last_peered":"2024-09-14T04:07:30.856487+0000","last_clean":"2024-09-14T04:07:30.856487+0000","last_became_active":"2024-09-14T04:04:00.931786+0000","last_became_peered":"2024-09-14T04:04:00.931786+0000","last_unstale":"2024-09-14T04:07:30.856487+0000","last_undegraded":"2024-09-14T04:07:30.856487+0000","last_fullsized":"2024-09-14T04:07:30.856487+0000","mapping_epoch":135,"log_start":"143'6221","ondisk_log_start":"143'6221","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:47.607999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13991,"num_write_kb":532160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.068904+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:07:24.068904+0000","last_peered":"2024-09-14T04:07:24.068904+0000","last_clean":"2024-09-14T04:07:24.068904+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:07:24.068904+0000","last_undegraded":"2024-09-14T04:07:24.068904+0000","last_fullsized":"2024-09-14T04:07:24.068904+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":151,"reported_epoch":164,"state":"active+undersized","last_fresh":"2024-09-14T04:07:21.038153+0000","last_change":"2024-09-14T04:06:23.181986+0000","last_active":"2024-09-14T04:07:21.038153+0000","last_peered":"2024-09-14T04:07:21.038153+0000","last_clean":"2024-09-14T04:06:19.060989+0000","last_became_active":"2024-09-14T04:06:23.181986+0000","last_became_peered":"2024-09-14T04:06:23.181986+0000","last_unstale":"2024-09-14T04:07:21.038153+0000","last_undegraded":"2024-09-14T04:07:21.038153+0000","last_fullsized":"2024-09-14T04:06:23.104250+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:54.913898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":164,"state":"active+undersized","last_fresh":"2024-09-14T04:07:21.029936+0000","last_change":"2024-09-14T04:06:23.108453+0000","last_active":"2024-09-14T04:07:21.029936+0000","last_peered":"2024-09-14T04:07:21.029936+0000","last_clean":"2024-09-14T03:42:34.625113+0000","last_became_active":"2024-09-14T04:06:23.108453+0000","last_became_peered":"2024-09-14T04:06:23.108453+0000","last_unstale":"2024-09-14T04:07:21.029936+0000","last_undegraded":"2024-09-14T04:07:21.029936+0000","last_fullsized":"2024-09-14T04:06:23.098104+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:40.264955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"165'13803","reported_seq":13844,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:30.850029+0000","last_change":"2024-09-14T04:04:00.899156+0000","last_active":"2024-09-14T04:07:30.850029+0000","last_peered":"2024-09-14T04:07:30.850029+0000","last_clean":"2024-09-14T04:07:30.850029+0000","last_became_active":"2024-09-14T04:04:00.898841+0000","last_became_peered":"2024-09-14T04:04:00.898841+0000","last_unstale":"2024-09-14T04:07:30.850029+0000","last_undegraded":"2024-09-14T04:07:30.850029+0000","last_fullsized":"2024-09-14T04:07:30.850029+0000","mapping_epoch":135,"log_start":"142'6021","ondisk_log_start":"142'6021","created":135,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:47:17.172140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178257944,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13803,"num_write_kb":528705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":60,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.039146+0000","last_change":"2024-09-14T04:07:21.038362+0000","last_active":"2024-09-14T04:07:21.039146+0000","last_peered":"2024-09-14T04:07:21.039146+0000","last_clean":"2024-09-14T04:07:21.039146+0000","last_became_active":"2024-09-14T04:07:21.037551+0000","last_became_peered":"2024-09-14T04:07:21.037551+0000","last_unstale":"2024-09-14T04:07:21.039146+0000","last_undegraded":"2024-09-14T04:07:21.039146+0000","last_fullsized":"2024-09-14T04:07:21.039146+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:49.355895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,5],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":150,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.070072+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:07:24.070072+0000","last_peered":"2024-09-14T04:07:24.070072+0000","last_clean":"2024-09-14T04:07:24.070072+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:07:24.070072+0000","last_undegraded":"2024-09-14T04:07:24.070072+0000","last_fullsized":"2024-09-14T04:07:24.070072+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.046545+0000","last_change":"2024-09-14T04:07:21.045185+0000","last_active":"2024-09-14T04:07:21.046545+0000","last_peered":"2024-09-14T04:07:21.046545+0000","last_clean":"2024-09-14T04:07:21.046545+0000","last_became_active":"2024-09-14T04:07:21.044776+0000","last_became_peered":"2024-09-14T04:07:21.044776+0000","last_unstale":"2024-09-14T04:07:21.046545+0000","last_undegraded":"2024-09-14T04:07:21.046545+0000","last_fullsized":"2024-09-14T04:07:21.046545+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:14.854057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"165'13900","reported_seq":14891,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:30.473333+0000","last_change":"2024-09-14T04:07:20.077105+0000","last_active":"2024-09-14T04:07:30.473333+0000","last_peered":"2024-09-14T04:07:30.473333+0000","last_clean":"2024-09-14T04:07:30.473333+0000","last_became_active":"2024-09-14T04:07:20.031512+0000","last_became_peered":"2024-09-14T04:07:20.031512+0000","last_unstale":"2024-09-14T04:07:30.473333+0000","last_undegraded":"2024-09-14T04:07:30.473333+0000","last_fullsized":"2024-09-14T04:07:30.473333+0000","mapping_epoch":162,"log_start":"142'5444","ondisk_log_start":"142'5444","created":135,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:03:59.881439+0000","last_clean_scrub_stamp":"2024-09-14T04:03:59.881439+0000","objects_scrubbed":0,"log_size":8456,"log_dups_size":0,"ondisk_log_size":8456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:59.503104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171966464,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13900,"num_write_kb":528768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":451,"num_bytes_recovered":29556736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":62,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:24.069932+0000","last_change":"2024-09-14T04:06:23.111706+0000","last_active":"2024-09-14T04:07:24.069932+0000","last_peered":"2024-09-14T04:07:24.069932+0000","last_clean":"2024-09-14T03:42:37.674390+0000","last_became_active":"2024-09-14T04:06:23.111706+0000","last_became_peered":"2024-09-14T04:06:23.111706+0000","last_unstale":"2024-09-14T04:07:24.069932+0000","last_undegraded":"2024-09-14T04:06:23.099338+0000","last_fullsized":"2024-09-14T04:06:23.098941+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:07.419642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":143,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.036301+0000","last_change":"2024-09-14T04:01:59.055291+0000","last_active":"2024-09-14T04:07:21.036301+0000","last_peered":"2024-09-14T04:07:21.036301+0000","last_clean":"2024-09-14T04:07:21.036301+0000","last_became_active":"2024-09-14T04:01:59.054919+0000","last_became_peered":"2024-09-14T04:01:59.054919+0000","last_unstale":"2024-09-14T04:07:21.036301+0000","last_undegraded":"2024-09-14T04:07:21.036301+0000","last_fullsized":"2024-09-14T04:07:21.036301+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:22.638368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.031312+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:07:21.031312+0000","last_peered":"2024-09-14T04:07:21.031312+0000","last_clean":"2024-09-14T04:07:21.031312+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:07:21.031312+0000","last_undegraded":"2024-09-14T04:07:21.031312+0000","last_fullsized":"2024-09-14T04:07:21.031312+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":139,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.032523+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:07:21.032523+0000","last_peered":"2024-09-14T04:07:21.032523+0000","last_clean":"2024-09-14T04:07:21.032523+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:07:21.032523+0000","last_undegraded":"2024-09-14T04:07:21.032523+0000","last_fullsized":"2024-09-14T04:07:21.032523+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":139,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.033680+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:07:21.033680+0000","last_peered":"2024-09-14T04:07:21.033680+0000","last_clean":"2024-09-14T04:07:21.033680+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:07:21.033680+0000","last_undegraded":"2024-09-14T04:07:21.033680+0000","last_fullsized":"2024-09-14T04:07:21.033680+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":150,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.069463+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:07:24.069463+0000","last_peered":"2024-09-14T04:07:24.069463+0000","last_clean":"2024-09-14T04:07:24.069463+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:07:24.069463+0000","last_undegraded":"2024-09-14T04:07:24.069463+0000","last_fullsized":"2024-09-14T04:07:24.069463+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":170,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.035785+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:07:21.035785+0000","last_peered":"2024-09-14T04:07:21.035785+0000","last_clean":"2024-09-14T04:07:21.035785+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:07:21.035785+0000","last_undegraded":"2024-09-14T04:07:21.035785+0000","last_fullsized":"2024-09-14T04:07:21.035785+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:18.015681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.029956+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:07:21.029956+0000","last_peered":"2024-09-14T04:07:21.029956+0000","last_clean":"2024-09-14T04:07:21.029956+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:07:21.029956+0000","last_undegraded":"2024-09-14T04:07:21.029956+0000","last_fullsized":"2024-09-14T04:07:21.029956+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.027030+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:07:21.027030+0000","last_peered":"2024-09-14T04:07:21.027030+0000","last_clean":"2024-09-14T04:07:21.027030+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:07:21.027030+0000","last_undegraded":"2024-09-14T04:07:21.027030+0000","last_fullsized":"2024-09-14T04:07:21.027030+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":157,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.031266+0000","last_change":"2024-09-14T04:06:47.339624+0000","last_active":"2024-09-14T04:07:21.031266+0000","last_peered":"2024-09-14T04:07:21.031266+0000","last_clean":"2024-09-14T04:07:21.031266+0000","last_became_active":"2024-09-14T04:06:47.339178+0000","last_became_peered":"2024-09-14T04:06:47.339178+0000","last_unstale":"2024-09-14T04:07:21.031266+0000","last_undegraded":"2024-09-14T04:07:21.031266+0000","last_fullsized":"2024-09-14T04:07:21.031266+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:55.489018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":28,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.062524+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:07:24.062524+0000","last_peered":"2024-09-14T04:07:24.062524+0000","last_clean":"2024-09-14T04:07:24.062524+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:07:24.062524+0000","last_undegraded":"2024-09-14T04:07:24.062524+0000","last_fullsized":"2024-09-14T04:07:24.062524+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":129,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.036237+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:07:21.036237+0000","last_peered":"2024-09-14T04:07:21.036237+0000","last_clean":"2024-09-14T04:07:21.036237+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:07:21.036237+0000","last_undegraded":"2024-09-14T04:07:21.036237+0000","last_fullsized":"2024-09-14T04:07:21.036237+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":165,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:24.068700+0000","last_change":"2024-09-14T04:07:21.053003+0000","last_active":"2024-09-14T04:07:24.068700+0000","last_peered":"2024-09-14T04:07:24.068700+0000","last_clean":"2024-09-14T04:07:24.068700+0000","last_became_active":"2024-09-14T04:07:21.052033+0000","last_became_peered":"2024-09-14T04:07:21.052033+0000","last_unstale":"2024-09-14T04:07:24.068700+0000","last_undegraded":"2024-09-14T04:07:24.068700+0000","last_fullsized":"2024-09-14T04:07:24.068700+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:56:19.771880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,2],"avail_no_missing":["1","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":159,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.027207+0000","last_change":"2024-09-14T04:06:47.336954+0000","last_active":"2024-09-14T04:07:21.027207+0000","last_peered":"2024-09-14T04:07:21.027207+0000","last_clean":"2024-09-14T04:07:21.027207+0000","last_became_active":"2024-09-14T04:06:47.336461+0000","last_became_peered":"2024-09-14T04:06:47.336461+0000","last_unstale":"2024-09-14T04:07:21.027207+0000","last_undegraded":"2024-09-14T04:07:21.027207+0000","last_fullsized":"2024-09-14T04:07:21.027207+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:41.289152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.068494+0000","last_change":"2024-09-14T04:06:46.341686+0000","last_active":"2024-09-14T04:07:24.068494+0000","last_peered":"2024-09-14T04:07:24.068494+0000","last_clean":"2024-09-14T04:07:24.068494+0000","last_became_active":"2024-09-14T04:06:46.341228+0000","last_became_peered":"2024-09-14T04:06:46.341228+0000","last_unstale":"2024-09-14T04:07:24.068494+0000","last_undegraded":"2024-09-14T04:07:24.068494+0000","last_fullsized":"2024-09-14T04:07:24.068494+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:39.010812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.032907+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:07:21.032907+0000","last_peered":"2024-09-14T04:07:21.032907+0000","last_clean":"2024-09-14T04:07:21.032907+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:07:21.032907+0000","last_undegraded":"2024-09-14T04:07:21.032907+0000","last_fullsized":"2024-09-14T04:07:21.032907+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":54,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.069675+0000","last_change":"2024-09-14T04:06:23.103876+0000","last_active":"2024-09-14T04:07:24.069675+0000","last_peered":"2024-09-14T04:07:24.069675+0000","last_clean":"2024-09-14T03:57:06.051230+0000","last_became_active":"2024-09-14T04:06:23.103876+0000","last_became_peered":"2024-09-14T04:06:23.103876+0000","last_unstale":"2024-09-14T04:07:24.069675+0000","last_undegraded":"2024-09-14T04:07:24.069675+0000","last_fullsized":"2024-09-14T04:06:23.099139+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:54.098236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-09-14T04:07:24.064425+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:07:24.064425+0000","last_peered":"2024-09-14T04:07:24.064425+0000","last_clean":"2024-09-14T04:07:24.064425+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:07:24.064425+0000","last_undegraded":"2024-09-14T04:07:24.064425+0000","last_fullsized":"2024-09-14T04:07:24.064425+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":53,"reported_epoch":165,"state":"active+undersized","last_fresh":"2024-09-14T04:07:24.063901+0000","last_change":"2024-09-14T04:06:23.110483+0000","last_active":"2024-09-14T04:07:24.063901+0000","last_peered":"2024-09-14T04:07:24.063901+0000","last_clean":"2024-09-14T03:57:06.051044+0000","last_became_active":"2024-09-14T04:06:23.110483+0000","last_became_peered":"2024-09-14T04:06:23.110483+0000","last_unstale":"2024-09-14T04:07:24.063901+0000","last_undegraded":"2024-09-14T04:07:24.063901+0000","last_fullsized":"2024-09-14T04:06:23.097209+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:58.082927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.032457+0000","last_change":"2024-09-14T04:06:46.339807+0000","last_active":"2024-09-14T04:07:21.032457+0000","last_peered":"2024-09-14T04:07:21.032457+0000","last_clean":"2024-09-14T04:07:21.032457+0000","last_became_active":"2024-09-14T04:06:46.339419+0000","last_became_peered":"2024-09-14T04:06:46.339419+0000","last_unstale":"2024-09-14T04:07:21.032457+0000","last_undegraded":"2024-09-14T04:07:21.032457+0000","last_fullsized":"2024-09-14T04:07:21.032457+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:18:31.877236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":169,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.030365+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:07:21.030365+0000","last_peered":"2024-09-14T04:07:21.030365+0000","last_clean":"2024-09-14T04:07:21.030365+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:07:21.030365+0000","last_undegraded":"2024-09-14T04:07:21.030365+0000","last_fullsized":"2024-09-14T04:07:21.030365+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":169,"reported_epoch":164,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:21.031542+0000","last_change":"2024-09-14T04:06:47.332225+0000","last_active":"2024-09-14T04:07:21.031542+0000","last_peered":"2024-09-14T04:07:21.031542+0000","last_clean":"2024-09-14T04:07:21.031542+0000","last_became_active":"2024-09-14T04:06:47.331786+0000","last_became_peered":"2024-09-14T04:06:47.331786+0000","last_unstale":"2024-09-14T04:07:21.031542+0000","last_undegraded":"2024-09-14T04:07:21.031542+0000","last_fullsized":"2024-09-14T04:07:21.031542+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:47.048287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,2],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":147,"reported_epoch":164,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:21.033225+0000","last_change":"2024-09-14T04:06:23.104756+0000","last_active":"2024-09-14T04:07:21.033225+0000","last_peered":"2024-09-14T04:07:21.033225+0000","last_clean":"2024-09-14T04:06:19.084001+0000","last_became_active":"2024-09-14T04:06:23.104756+0000","last_became_peered":"2024-09-14T04:06:23.104756+0000","last_unstale":"2024-09-14T04:07:21.033225+0000","last_undegraded":"2024-09-14T04:06:23.097257+0000","last_fullsized":"2024-09-14T04:06:23.097149+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:28.712625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":148,"reported_epoch":165,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:07:24.063875+0000","last_change":"2024-09-14T04:07:21.044184+0000","last_active":"2024-09-14T04:07:24.063875+0000","last_peered":"2024-09-14T04:07:24.063875+0000","last_clean":"2024-09-14T04:07:24.063875+0000","last_became_active":"2024-09-14T04:07:21.043623+0000","last_became_peered":"2024-09-14T04:07:21.043623+0000","last_unstale":"2024-09-14T04:07:24.063875+0000","last_undegraded":"2024-09-14T04:07:24.063875+0000","last_fullsized":"2024-09-14T04:07:24.063875+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:57:10.731246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":38,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.036047+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:07:21.036047+0000","last_peered":"2024-09-14T04:07:21.036047+0000","last_clean":"2024-09-14T04:07:21.036047+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:07:21.036047+0000","last_undegraded":"2024-09-14T04:07:21.036047+0000","last_fullsized":"2024-09-14T04:07:21.036047+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":130,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-09-14T04:07:21.030997+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:07:21.030997+0000","last_peered":"2024-09-14T04:07:21.030997+0000","last_clean":"2024-09-14T04:07:21.030997+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:07:21.030997+0000","last_undegraded":"2024-09-14T04:07:21.030997+0000","last_fullsized":"2024-09-14T04:07:21.030997+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":49,"reported_epoch":164,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:07:21.031149+0000","last_change":"2024-09-14T04:06:23.102497+0000","last_active":"2024-09-14T04:07:21.031149+0000","last_peered":"2024-09-14T04:07:21.031149+0000","last_clean":"2024-09-14T03:42:37.674401+0000","last_became_active":"2024-09-14T04:06:23.102497+0000","last_became_peered":"2024-09-14T04:06:23.102497+0000","last_unstale":"2024-09-14T04:07:21.031149+0000","last_undegraded":"2024-09-14T04:06:23.098430+0000","last_fullsized":"2024-09-14T04:06:23.098319+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:11.642487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2785017880,"num_objects":42497,"num_object_clones":0,"num_object_copies":127491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42497,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":220687,"num_write_kb":8421825,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10129,"num_bytes_recovered":663224320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9454759936,"data_stored":9454747720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131557,"ondisk_log_size":131557,"up":44,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":92,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13714,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":80,"acting":86,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":20,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":2,"acting":2,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201603,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2331724,"kb_used_data":1660484,"kb_used_omap":1,"kb_used_meta":670974,"kb_avail":91413428,"statfs":{"total":95995035648,"available":93607350272,"internally_reserved":0,"allocated":1700335616,"data_stored":1699299565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":687077835},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463241,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035196,"kb_used_data":512692,"kb_used_omap":1,"kb_used_meta":522494,"kb_avail":92709956,"statfs":{"total":95995035648,"available":94934994944,"internally_reserved":0,"allocated":524996608,"data_stored":523976270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":535034311},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724877,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2396216,"kb_used_data":1570864,"kb_used_omap":1,"kb_used_meta":825214,"kb_avail":91348936,"statfs":{"total":95995035648,"available":93541310464,"internally_reserved":0,"allocated":1608564736,"data_stored":1607544073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":845019590},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986513,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169728,"kb_used_data":520888,"kb_used_omap":1,"kb_used_meta":648830,"kb_avail":92575424,"statfs":{"total":95995035648,"available":94797234176,"internally_reserved":0,"allocated":533389312,"data_stored":532364672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":664402374},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748542,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564656,"kb_used_data":1342440,"kb_used_omap":4,"kb_used_meta":222203,"kb_avail":92180496,"statfs":{"total":95995035648,"available":94392827904,"internally_reserved":0,"allocated":1374658560,"data_stored":1373645777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":227536428},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912069,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":631988,"kb_used_data":490924,"kb_used_omap":6,"kb_used_meta":141049,"kb_avail":93113164,"statfs":{"total":95995035648,"available":95347879936,"internally_reserved":0,"allocated":502706176,"data_stored":501689823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":144434745},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468085,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989744,"kb_used_data":1588392,"kb_used_omap":5,"kb_used_meta":401210,"kb_avail":91755408,"statfs":{"total":95995035648,"available":93957537792,"internally_reserved":0,"allocated":1626513408,"data_stored":1625499158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":410839391},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":102,"seq":438086664305,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1880608,"kb_used_data":1663448,"kb_used_omap":5,"kb_used_meta":217146,"kb_avail":91864544,"statfs":{"total":95995035648,"available":94069293056,"internally_reserved":0,"allocated":1703370752,"data_stored":1702447034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5976,"internal_metadata":222357672},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4382,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1695088640,"data_stored":1695088640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1608978432,"data_stored":1608974360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":499449856,"data_stored":499449856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1360068608,"data_stored":1360068608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":521404416,"data_stored":521404416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1574703104,"data_stored":1574699032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":521535488,"data_stored":521535488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1673531392,"data_stored":1673527320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:07:36.264 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:07:36.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-14T04:07:36.265 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:36 smithi001 ceph-mon[107399]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:07:36.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:36.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:36 smithi012 ceph-mon[101180]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:07:36.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:36.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:36.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:36.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:36.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:36.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:36.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:36.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:36.873 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:36.904 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:36.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:36.964 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:36] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.51.0" 2024-09-14T04:07:37.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:37.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:37.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:37.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:37 smithi001 ceph-mon[107399]: pgmap v586: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 10318/123003 objects degraded (8.388%); 2/123003 objects misplaced (0.002%) 2024-09-14T04:07:37.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:37 smithi001 ceph-mon[112165]: pgmap v586: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 10318/123003 objects degraded (8.388%); 2/123003 objects misplaced (0.002%) 2024-09-14T04:07:37.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:37.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:37.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:37.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:37.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:37.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:37 smithi012 ceph-mon[101180]: pgmap v586: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 10318/123003 objects degraded (8.388%); 2/123003 objects misplaced (0.002%) 2024-09-14T04:07:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.619 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:37.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:37.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:37.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:37.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:37.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:37.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:37.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:38.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:38.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:38.117 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:07:38.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:38.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:38.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:38.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:38.367 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:38.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:38.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:07:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:07:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:38.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:07:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:38.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:38.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:38.743 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:38.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:38.792 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:38.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:38.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:38.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:39.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:39.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:39.164 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:07:39.164 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:07:39.164 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:07:39.164 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:07:39.165 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:07:39.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:07:39.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:07:39.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:07:39.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:07:39.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:39.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:39.202 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:07:39.202 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:07:39.227 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:07:39.227 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:07:39.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:39.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:39.329 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:07:39.330 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:07:39.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: pgmap v587: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 9429/113055 objects degraded (8.340%); 2/113055 objects misplaced (0.002%) 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: osdmap e166: 8 total, 7 up, 8 in 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9429/113055 objects degraded (8.340%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1412898741' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: pgmap v587: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 9429/113055 objects degraded (8.340%); 2/113055 objects misplaced (0.002%) 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: osdmap e166: 8 total, 7 up, 8 in 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9429/113055 objects degraded (8.340%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:39.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1412898741' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:39.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:39.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: pgmap v587: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 9429/113055 objects degraded (8.340%); 2/113055 objects misplaced (0.002%) 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/688062287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: osdmap e166: 8 total, 7 up, 8 in 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:39.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9429/113055 objects degraded (8.340%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:39.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1412898741' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:39.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:39.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:39.730 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:39.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:39.864 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:39.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:39.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:40.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:40.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:40.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:40.255 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:40.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:40.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:40.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:40.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:40.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:40.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:40.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:40.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:40.811 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:40.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:40.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:07:40.983 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:07:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:40.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:40.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:41.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:41.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:41.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:41.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:41.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:41 smithi001 ceph-mon[107399]: pgmap v589: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 9429/111459 objects degraded (8.460%); 2/111459 objects misplaced (0.002%) 2024-09-14T04:07:41.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:41 smithi001 ceph-mon[112165]: pgmap v589: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 9429/111459 objects degraded (8.460%); 2/111459 objects misplaced (0.002%) 2024-09-14T04:07:41.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:41.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:41.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:41.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:41.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:41.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:41 smithi012 ceph-mon[101180]: pgmap v589: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 9429/111459 objects degraded (8.460%); 2/111459 objects misplaced (0.002%) 2024-09-14T04:07:41.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:41.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:41.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:41.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:41.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:41.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:41.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:41.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:42.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:42.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:42.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:42.222 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:42.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:42.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:42.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:42.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:07:42.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.512 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:42.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:42.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:42.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:42.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:42.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:42.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:42.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:42.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:42.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:07:43.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:43.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:43.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:43.192 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:43.247 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:07:43.248 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:07:43.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:43.303 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:43 smithi001 ceph-mon[107399]: pgmap v590: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:43.303 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:43 smithi001 ceph-mon[112165]: pgmap v590: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:43.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:43.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:43.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:43.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:43.560 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:43 smithi012 ceph-mon[101180]: pgmap v590: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:43.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:43.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:43.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:43.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:43.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:43.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:43.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:44.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:44.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:44.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:44.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:44.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:44.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:44.294 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:44.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:07:44.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8248/99471 objects degraded (8.292%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:44.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8248/99471 objects degraded (8.292%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:44.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:44.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:44.470 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8248/99471 objects degraded (8.292%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:44.538 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:44.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:44.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:44.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:44.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:44.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:44.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:44.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:45.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:45.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:45.134 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:45.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:45 smithi001 ceph-mon[107399]: pgmap v591: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:45.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:45 smithi001 ceph-mon[112165]: pgmap v591: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:45.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:45.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:45.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:45.438 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:45 smithi012 ceph-mon[101180]: pgmap v591: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 8248/99471 objects degraded (8.292%); 2/99471 objects misplaced (0.002%) 2024-09-14T04:07:45.439 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:45.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:45.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:45.628 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:45.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:45.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:45.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:45.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:45.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:46.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:46.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:46.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:46.158 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 1 pg_num to 11 2024-09-14T04:07:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:46.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:46.290 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:46.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:46.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:46.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/484941606' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/484941606' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.472 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:46.524 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/484941606' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.524 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:07:46.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:46.609 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:46.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:46.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:46.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:46.859 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:46] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.51.0" 2024-09-14T04:07:46.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:46.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:46.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:47.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:47.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:47.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:47.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:47.251 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:47.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:47.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:47.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:47 smithi012 ceph-mon[101180]: pgmap v592: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 7967/95244 objects degraded (8.365%); 2/95244 objects misplaced (0.002%) 2024-09-14T04:07:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:47 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:07:47.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:47 smithi012 ceph-mon[101180]: osdmap e167: 8 total, 7 up, 8 in 2024-09-14T04:07:47.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:47.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[107399]: pgmap v592: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 7967/95244 objects degraded (8.365%); 2/95244 objects misplaced (0.002%) 2024-09-14T04:07:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:07:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[107399]: osdmap e167: 8 total, 7 up, 8 in 2024-09-14T04:07:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[112165]: pgmap v592: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 7967/95244 objects degraded (8.365%); 2/95244 objects misplaced (0.002%) 2024-09-14T04:07:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:07:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:47 smithi001 ceph-mon[112165]: osdmap e167: 8 total, 7 up, 8 in 2024-09-14T04:07:47.478 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:47.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:47.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:47.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:47.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:47.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:47.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:48.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:48.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:48.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:48.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:48.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:48.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:48.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:48.421 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:07:48.421 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:07:48.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:48.476 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:48.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:07:48.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:48.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:48.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:48.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:48.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:49.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:49.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:49.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:49.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:49.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:49.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: pgmap v594: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 7131/86229 objects degraded (8.270%); 2/86229 objects misplaced (0.002%) 2024-09-14T04:07:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:07:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: osdmap e168: 8 total, 7 up, 8 in 2024-09-14T04:07:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.413 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7131/86229 objects degraded (8.270%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: pgmap v594: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 7131/86229 objects degraded (8.270%); 2/86229 objects misplaced (0.002%) 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: osdmap e168: 8 total, 7 up, 8 in 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.414 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7131/86229 objects degraded (8.270%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:49.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: pgmap v594: 89 pgs: 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 7131/86229 objects degraded (8.270%); 2/86229 objects misplaced (0.002%) 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: osdmap e168: 8 total, 7 up, 8 in 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7131/86229 objects degraded (8.270%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:49.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:49.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:49.753 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:49.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:49.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:49.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:49.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:50.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:50.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:50.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:50.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:50.342 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:50.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[107399]: osdmap e169: 8 total, 7 up, 8 in 2024-09-14T04:07:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:50.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[112165]: osdmap e169: 8 total, 7 up, 8 in 2024-09-14T04:07:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:50.474 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:50.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:50.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:50 smithi012 ceph-mon[101180]: osdmap e169: 8 total, 7 up, 8 in 2024-09-14T04:07:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:07:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:50.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:50.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:50.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:50.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:50.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:50.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:51.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:51.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:51.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:51.179 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:07:51.183 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:51.285 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:51.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:51.421 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:51.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:51.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:51 smithi001 ceph-mon[107399]: pgmap v597: 99 pgs: 10 unknown, 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7131/84594 objects degraded (8.430%); 2/84594 objects misplaced (0.002%) 2024-09-14T04:07:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:51 smithi001 ceph-mon[112165]: pgmap v597: 99 pgs: 10 unknown, 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7131/84594 objects degraded (8.430%); 2/84594 objects misplaced (0.002%) 2024-09-14T04:07:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:51.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:51.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:51 smithi012 ceph-mon[101180]: pgmap v597: 99 pgs: 10 unknown, 12 active+clean+remapped, 11 active+undersized, 11 active+undersized+degraded, 55 active+clean; 1.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7131/84594 objects degraded (8.430%); 2/84594 objects misplaced (0.002%) 2024-09-14T04:07:51.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:07:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:51.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:51.709 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:51.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:51.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:51.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:51.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:51.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:52.037 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:52.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:52.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:52.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:52.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:52.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:52.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[107399]: osdmap e170: 8 total, 7 up, 8 in 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:52.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:52 smithi001 ceph-mon[112165]: osdmap e170: 8 total, 7 up, 8 in 2024-09-14T04:07:52.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:52 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:07:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3695195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:07:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:52 smithi012 ceph-mon[101180]: osdmap e170: 8 total, 7 up, 8 in 2024-09-14T04:07:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.630 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:52.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:52.673 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:07:52.705 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:07:52.706 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:07:52.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:52.739 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:07:52.739 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:07:52.765 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:52.769 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:07:52.769 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:07:52.799 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:07:52.799 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:07:52.829 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:07:52.830 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:07:52.861 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:07:52.862 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:07:52.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:52.894 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:07:52.894 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:07:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:52.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:52.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:53.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:53.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:53.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:53.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:53.221 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:07:53.222 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:07:53.222 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-09-14T04:07:53.222 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-09-14T04:07:53.222 DEBUG:teuthology.orchestra.run.smithi001:osd.0> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service 2024-09-14T04:07:53.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl start ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0 2024-09-14T04:07:53.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:53.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:53.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:53 smithi001 ceph-mon[107399]: pgmap v599: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/678404701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:53 smithi001 ceph-mon[112165]: pgmap v599: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/678404701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:53.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:53 smithi001 systemd[1]: Starting Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:53.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:53.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:53.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:53 smithi012 ceph-mon[101180]: pgmap v599: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:53.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/678404701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:07:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.629 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:53.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:53.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:53.770 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:53.872 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:53.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:53.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:54.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:54.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:54.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:54.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:54.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:54.288 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:54.325 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:54.427 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:54.454 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:54.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5910/71991 objects degraded (8.209%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:54.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5910/71991 objects degraded (8.209%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:54.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:54.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5910/71991 objects degraded (8.209%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:54.604 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:07:54.604 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:07:54.605 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:07:54.605 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:07:54.605 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:07:54.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:54.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:54.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:54.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:54.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:54.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:54.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:55.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:55.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:55.154 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:55.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:55.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:55.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:55.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:55.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:55 smithi001 ceph-mon[107399]: pgmap v600: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:55 smithi001 ceph-mon[112165]: pgmap v600: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:55.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:55.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:55 smithi012 ceph-mon[101180]: pgmap v600: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5910/71991 objects degraded (8.209%); 2/71991 objects misplaced (0.003%) 2024-09-14T04:07:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:07:55.641 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:07:55.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:55.716 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:55.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:55.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:55.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:55.993 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:55.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:56.095 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:56.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:56.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:56.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:56.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:56.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:56.472 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:56 smithi001 podman[182208]: 2024-09-14 04:07:56.199795061 +0000 UTC m=+0.026549178 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:56.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:56.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:56.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:56.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:56.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:07:56.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:56.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:56.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:56.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:56.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:56.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:07:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:07:56] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.51.0" 2024-09-14T04:07:56.973 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:56 smithi001 podman[182208]: 2024-09-14 04:07:56.861988163 +0000 UTC m=+0.688742266 container create 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T04:07:57.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:57.088 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:57.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:57.238 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:07:57.323 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:57.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:57.360 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:57 smithi001 ceph-mon[112165]: pgmap v601: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5568/67281 objects degraded (8.276%); 2/67281 objects misplaced (0.003%) 2024-09-14T04:07:57.361 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 podman[182208]: 2024-09-14 04:07:57.116491525 +0000 UTC m=+0.943245637 container init 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-14T04:07:57.361 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 podman[182208]: 2024-09-14 04:07:57.126375359 +0000 UTC m=+0.953129470 container start 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T04:07:57.361 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 podman[182208]: 2024-09-14 04:07:57.223582581 +0000 UTC m=+1.050336692 container attach 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T04:07:57.361 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:57.361 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 bash[182208]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:57.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:57 smithi001 ceph-mon[107399]: pgmap v601: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5568/67281 objects degraded (8.276%); 2/67281 objects misplaced (0.003%) 2024-09-14T04:07:57.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:57.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:57.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:57.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:57 smithi012 ceph-mon[101180]: pgmap v601: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5568/67281 objects degraded (8.276%); 2/67281 objects misplaced (0.003%) 2024-09-14T04:07:57.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:57.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:07:57.653 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:57.653 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 bash[182208]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:57.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:57.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:57.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:57.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:57.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:57.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:57.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:57.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:58.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:07:58.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:07:58.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:07:58.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:57 smithi001 bash[182208]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:07:58.222 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:07:58.223 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate[182341]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T04:07:58.223 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:58 smithi001 bash[182208]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T04:07:58.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:58.264 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:58.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:58.408 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:58.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:58.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:58.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:58.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:58.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:07:58.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:07:58.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:58.953 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:58.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:59.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:07:59.093 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:07:59.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:59.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:59.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:07:59.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:07:59.384 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:59 smithi001 ceph-mon[112165]: pgmap v602: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 4644/56871 objects degraded (8.166%); 2/56871 objects misplaced (0.004%) 2024-09-14T04:07:59.384 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:07:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4644/56871 objects degraded (8.166%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:59.384 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:59 smithi001 ceph-mon[107399]: pgmap v602: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 4644/56871 objects degraded (8.166%); 2/56871 objects misplaced (0.004%) 2024-09-14T04:07:59.384 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:07:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4644/56871 objects degraded (8.166%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:59.384 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:59 smithi001 podman[182208]: 2024-09-14 04:07:59.046946523 +0000 UTC m=+2.873700647 container died 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T04:07:59.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:59.516 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:59 smithi012 ceph-mon[101180]: pgmap v602: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 4644/56871 objects degraded (8.166%); 2/56871 objects misplaced (0.004%) 2024-09-14T04:07:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:07:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4644/56871 objects degraded (8.166%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:07:59.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:07:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.661 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:07:59.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:07:59.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:07:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:07:59.858 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:07:59.900 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:07:59.930 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:07:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:07:59.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:00.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:00.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:00.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:00.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:00.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:07:59 smithi001 podman[182795]: 2024-09-14 04:07:59.817264543 +0000 UTC m=+1.693952565 container remove 466c14ca65455a63336e7c21b2e6bffa74350c8deebb721508956e430af2dafc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:08:00.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:00.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:00.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:00.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:00.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:00.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:00.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:00.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:00.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:00.904 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:00.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:00.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:01.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:01.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:01.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:01.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:01.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:01.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:01.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:01 smithi001 ceph-mon[107399]: pgmap v603: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4644/55014 objects degraded (8.441%); 2/55014 objects misplaced (0.004%) 2024-09-14T04:08:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:01 smithi001 ceph-mon[112165]: pgmap v603: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4644/55014 objects degraded (8.441%); 2/55014 objects misplaced (0.004%) 2024-09-14T04:08:01.475 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:01.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:01 smithi012 ceph-mon[101180]: pgmap v603: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4644/55014 objects degraded (8.441%); 2/55014 objects misplaced (0.004%) 2024-09-14T04:08:01.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:01.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:01.754 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:01.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:01.841 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:148259 9.1 9:86415656:::benchmark_data_smithi001_156540_object16665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:03.987231+0000", 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.308305828, 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.231216792, 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.842 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 148259 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987231+0000", 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987231+0000", 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987233+0000", 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5689999999999999e-06 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987239+0000", 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6359999999999999e-06 2024-09-14T04:08:01.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987251+0000", 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2199e-05 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987259+0000", 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6109999999999994e-06 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987318+0000", 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9045999999999997e-05 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987473+0000", 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015488599999999999 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987594+0000", 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120699 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.996597+0000", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0090031339999999994 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.996664+0000", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6972999999999997e-05 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218355+0000", 2024-09-14T04:08:01.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.221691157 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218393+0000", 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.807e-05 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218448+0000", 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4812000000000003e-05 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.848 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:148260 9.1 9:8b986ea2:::benchmark_data_smithi001_156540_object16666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:03.987940+0000", 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.30759695099999, 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23068254199999999, 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.849 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 148260 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987940+0000", 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987940+0000", 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987941+0000", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2099999999999996e-07 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987945+0000", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5650000000000002e-06 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987955+0000", 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0052e-05 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.987960+0000", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.711e-06 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.992424+0000", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044633839999999999 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.992541+0000", 2024-09-14T04:08:01.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117769 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.992643+0000", 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102064 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.996641+0000", 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039973789999999997 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:03.996670+0000", 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9583000000000001e-05 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218523+0000", 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22185295699999999 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218576+0000", 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2536e-05 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218623+0000", 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6820999999999998e-05 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:148279 9.1 9:831a50c7:::benchmark_data_smithi001_156540_object16685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:04.011550+0000", 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.283987087, 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20719305399999999, 2024-09-14T04:08:01.855 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 148279 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011550+0000", 2024-09-14T04:08:01.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011550+0000", 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011552+0000", 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.466e-06 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011557+0000", 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2039999999999996e-06 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011573+0000", 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6359999999999999e-05 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011580+0000", 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.063e-06 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011638+0000", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7787000000000001e-05 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011754+0000", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115771 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.011878+0000", 2024-09-14T04:08:01.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012463200000000001 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.015649+0000", 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037706469999999998 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.015705+0000", 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6010000000000001e-05 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218665+0000", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.202959996 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218709+0000", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3992999999999997e-05 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:04.218743+0000", 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4125000000000003e-05 2024-09-14T04:08:01.861 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178517 9.1 9:8c40454a:::benchmark_data_smithi001_156540_object46923:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331224+0000", 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964313277000002, 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0390826559999999, 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.862 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178517 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331224+0000", 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331224+0000", 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331225+0000", 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.437e-06 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331232+0000", 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1909999999999997e-06 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331244+0000", 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2283000000000001e-05 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331254+0000", 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7030000000000001e-06 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331307+0000", 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3298999999999999e-05 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331316+0000", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4360000000000008e-06 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.370225+0000", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0389086780000001 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.370307+0000", 2024-09-14T04:08:01.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1629000000000001e-05 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178520 9.8 9:1c2e04c6:::benchmark_data_smithi001_156540_object46926:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331272+0000", 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964265428000004, 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.03537579, 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.867 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178520 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331272+0000", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331272+0000", 2024-09-14T04:08:01.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331272+0000", 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9699999999999996e-07 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331276+0000", 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2279999999999998e-06 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331283+0000", 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7309999999999993e-06 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331289+0000", 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0589999999999999e-06 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331772+0000", 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048249600000000001 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331779+0000", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5340000000000002e-06 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.366573+0000", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0347934320000001 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.366648+0000", 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4713000000000001e-05 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.871 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178526 9.1 9:8b7b33d1:::benchmark_data_smithi001_156540_object46932:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331298+0000", 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964239082000006, 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0567422660000001, 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178526 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331298+0000", 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331298+0000", 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331299+0000", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1399999999999997e-07 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331301+0000", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2510000000000001e-06 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331308+0000", 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8929999999999999e-06 2024-09-14T04:08:01.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331313+0000", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4229999999999996e-06 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331342+0000", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8569e-05 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331348+0000", 2024-09-14T04:08:01.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1090000000000003e-06 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.387996+0000", 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0566485670000001 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388040+0000", 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4039999999999998e-05 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178531 9.1 9:8320b82a:::benchmark_data_smithi001_156540_object46937:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331321+0000", 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964215999000004, 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.075573731, 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178531 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331321+0000", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331321+0000", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331322+0000", 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9700000000000002e-07 2024-09-14T04:08:01.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331324+0000", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9290000000000001e-06 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331332+0000", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0809999999999994e-06 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331337+0000", 2024-09-14T04:08:01.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4949999999999999e-06 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331374+0000", 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7057000000000001e-05 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331380+0000", 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2430000000000002e-06 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388089+0000", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.056708507 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388161+0000", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1755000000000007e-05 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.406810+0000", 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018649464000000001 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.406895+0000", 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4802999999999994e-05 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178533 9.c 9:347ae011:::benchmark_data_smithi001_156540_object46939:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331344+0000", 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964193621999996, 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0513335079999999, 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.882 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178533 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331344+0000", 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331344+0000", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331344+0000", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0999999999999999e-07 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331346+0000", 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.052e-06 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331354+0000", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7959999999999996e-06 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331359+0000", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1830000000000002e-06 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331573+0000", 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021414000000000001 2024-09-14T04:08:01.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331583+0000", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5410000000000006e-06 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364603+0000", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0330198779999999 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364700+0000", 2024-09-14T04:08:01.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7394999999999998e-05 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.365308+0000", 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00060759099999999999 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.374030+0000", 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087223649999999993 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.377457+0000", 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034268269999999999 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382381+0000", 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0049243350000000002 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382558+0000", 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017648699999999999 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382677+0000", 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011950799999999999 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178535 9.1 9:830b5300:::benchmark_data_smithi001_156540_object46941:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331512+0000", 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.964025632000002, 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0749144450000001, 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.889 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178535 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331512+0000", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331512+0000", 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331512+0000", 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8999999999999995e-07 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331550+0000", 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7126999999999999e-05 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331560+0000", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0494e-05 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331568+0000", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5979999999999999e-06 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332169+0000", 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00060165000000000001 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332176+0000", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2859999999999996e-06 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388179+0000", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.056003596 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388260+0000", 2024-09-14T04:08:01.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1125999999999995e-05 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.388817+0000", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055663700000000002 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.401506+0000", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012688946 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.401520+0000", 2024-09-14T04:08:01.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.402e-05 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.406273+0000", 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047526189999999996 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.406329+0000", 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6713999999999999e-05 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.406426+0000", 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6742000000000003e-05 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178537 9.8 9:1fdcb056:::benchmark_data_smithi001_156540_object46943:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331584+0000", 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.963953029999999, 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0551322750000001, 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.896 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178537 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331584+0000", 2024-09-14T04:08:01.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331584+0000", 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331585+0000", 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4499999999999997e-07 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331588+0000", 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.96e-06 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331595+0000", 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9330000000000004e-06 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331601+0000", 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1449999999999996e-06 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332037+0000", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043610699999999998 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332045+0000", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5040000000000002e-06 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.367094+0000", 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.035048279 2024-09-14T04:08:01.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.367210+0000", 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011626800000000001 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.367786+0000", 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00057567299999999996 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386528+0000", 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018742413999999999 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386549+0000", 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0590000000000001e-05 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386601+0000", 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2317000000000001e-05 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386716+0000", 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011544000000000001 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178547 9.8 9:151e2423:::benchmark_data_smithi001_156540_object46953:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.331609+0000", 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.963928042000006, 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0553474249999999, 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.903 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178547 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331609+0000", 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331609+0000", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331610+0000", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1199999999999998e-07 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331612+0000", 2024-09-14T04:08:01.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1239999999999999e-06 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331619+0000", 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4189999999999996e-06 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.331624+0000", 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3499999999999996e-06 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332128+0000", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00050322099999999999 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.332135+0000", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0940000000000002e-06 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.376151+0000", 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.04401648 2024-09-14T04:08:01.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.376266+0000", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114286 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.376392+0000", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012656399999999999 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386729+0000", 2024-09-14T04:08:01.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010337209 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386787+0000", 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7487e-05 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386856+0000", 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9246999999999996e-05 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386957+0000", 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010053200000000001 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178549 9.2 9:49424544:::benchmark_data_smithi001_156540_object46955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.368418+0000", 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.927118789000005, 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0031654729999999, 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.910 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178549 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368418+0000", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368418+0000", 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368420+0000", 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4389999999999999e-06 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368425+0000", 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2639999999999999e-06 2024-09-14T04:08:01.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368436+0000", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1299e-05 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368443+0000", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1999999999999999e-06 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368486+0000", 2024-09-14T04:08:01.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3489999999999999e-05 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368493+0000", 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1600000000000001e-06 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.360645+0000", 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.992152221 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.360770+0000", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012427099999999999 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.360897+0000", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012723899999999999 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364422+0000", 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035245509999999999 2024-09-14T04:08:01.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371410+0000", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006988953 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371501+0000", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0327000000000004e-05 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371542+0000", 2024-09-14T04:08:01.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1588000000000002e-05 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371584+0000", 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1471000000000003e-05 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178554 9.2 9:4b2448fb:::benchmark_data_smithi001_156540_object46960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.426960+0000", 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.868576766000004, 2024-09-14T04:08:01.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94508034100000005, 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178554 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426960+0000", 2024-09-14T04:08:01.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426960+0000", 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426962+0000", 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.42e-06 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426967+0000", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0830000000000003e-06 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426978+0000", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0975000000000001e-05 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.426983+0000", 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.412e-06 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.427037+0000", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3699000000000002e-05 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.427046+0000", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1840000000000005e-06 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364463+0000", 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93741667200000001 2024-09-14T04:08:01.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364579+0000", 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115902 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.364690+0000", 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000111408 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371486+0000", 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0067955369999999999 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371664+0000", 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017796199999999999 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371955+0000", 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000291618 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372004+0000", 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8418000000000003e-05 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372041+0000", 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7051000000000002e-05 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178558 9.2 9:421ae772:::benchmark_data_smithi001_156540_object46964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.924 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.430438+0000", 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.865098891000002, 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94208491500000002, 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178558 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430438+0000", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430438+0000", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430440+0000", 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4470000000000001e-06 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430445+0000", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9420000000000003e-06 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430460+0000", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5430999999999998e-05 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430469+0000", 2024-09-14T04:08:01.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6070000000000008e-06 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430525+0000", 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5903999999999999e-05 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.430532+0000", 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8699999999999992e-06 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.368134+0000", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93760104200000005 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.368238+0000", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010427 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.368304+0000", 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5746000000000004e-05 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.371696+0000", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033926080000000001 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372080+0000", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038366399999999998 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372446+0000", 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00036569 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372490+0000", 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4110000000000003e-05 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.372523+0000", 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3584000000000002e-05 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.931 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178562 9.8 9:1bff70bc:::benchmark_data_smithi001_156540_object46968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.439519+0000", 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.856018657999996, 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95197657800000002, 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178562 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439519+0000", 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439519+0000", 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439520+0000", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.553e-06 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439526+0000", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0340000000000002e-06 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439541+0000", 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.467e-05 2024-09-14T04:08:01.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439549+0000", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1259999999999998e-06 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439601+0000", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1612e-05 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.439607+0000", 2024-09-14T04:08:01.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1149999999999999e-06 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.379406+0000", 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93979939899999998 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.379495+0000", 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8597999999999994e-05 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.379569+0000", 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4294000000000004e-05 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.386742+0000", 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007172678 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.391333+0000", 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045911420000000003 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.391372+0000", 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8871000000000001e-05 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.391495+0000", 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012348599999999999 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.938 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178569 9.c 9:381d8593:::benchmark_data_smithi001_156540_object46975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:46.470677+0000", 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 74.824860400000006, 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.91212666399999998, 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178569 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470677+0000", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470677+0000", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470678+0000", 2024-09-14T04:08:01.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7150000000000001e-06 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470685+0000", 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4200000000000004e-06 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470698+0000", 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3413e-05 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470709+0000", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0721e-05 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470767+0000", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7688000000000001e-05 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.470775+0000", 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5439999999999999e-06 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.374051+0000", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90327597599999998 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.374135+0000", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4217999999999999e-05 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.374223+0000", 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7726000000000007e-05 2024-09-14T04:08:01.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.377477+0000", 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032538100000000002 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.379343+0000", 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001866393 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.944 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382721+0000", 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033775099999999998 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382768+0000", 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7184000000000001e-05 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:47.382803+0000", 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5345999999999997e-05 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.945 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:180014 9.8 9:134ca631:::benchmark_data_smithi001_156540_object48420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.787872+0000", 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 72.507665402000001, 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.137708939, 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 180014 2024-09-14T04:08:01.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787872+0000", 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787872+0000", 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787873+0000", 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3230000000000001e-06 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787878+0000", 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.656e-06 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787894+0000", 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6464e-05 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.787901+0000", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1940000000000001e-06 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.789056+0000", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001154761 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.789169+0000", 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000112927 2024-09-14T04:08:01.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.789283+0000", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114281 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.796285+0000", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0070013920000000004 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925475+0000", 2024-09-14T04:08:01.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.12918976900000001 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925514+0000", 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9827999999999999e-05 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925581+0000", 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6344e-05 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:180015 9.8 9:1acee40b:::benchmark_data_smithi001_156540_object48421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.788028+0000", 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 72.507509462000002, 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.137806401, 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 180015 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.952 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788028+0000", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788028+0000", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788029+0000", 2024-09-14T04:08:01.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5000000000000001e-07 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788033+0000", 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.211e-06 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788045+0000", 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2486999999999999e-05 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.788052+0000", 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8630000000000002e-06 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.792797+0000", 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047452630000000004 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.792930+0000", 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000132409 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.793050+0000", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120705 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.796495+0000", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034443020000000002 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925642+0000", 2024-09-14T04:08:01.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.12914708699999999 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925708+0000", 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6295000000000001e-05 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.925834+0000", 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012592899999999999 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:180031 9.8 9:17986063:::benchmark_data_smithi001_156540_object48437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:48.802194+0000", 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 72.493343096999993, 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.131378198, 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 180031 2024-09-14T04:08:01.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802194+0000", 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802194+0000", 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802196+0000", 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.821e-06 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802202+0000", 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.314e-06 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802223+0000", 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0517e-05 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802231+0000", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5979999999999997e-06 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802747+0000", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000515486 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802856+0000", 2024-09-14T04:08:01.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108905 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.802971+0000", 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115709 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.806721+0000", 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037494329999999999 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:01.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.933389+0000", 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.126667894 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.933456+0000", 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7617e-05 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:48.933572+0000", 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011590399999999999 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.963 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:220056 9.1 e165/162 9:8f6b9db7:::benchmark_data_smithi001_156540_object88462:head v 165'13804, mlcod=165'13802)", 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:07:30.010452+0000", 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 31.285085125999998, 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13804097200000001, 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:08:01.964 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.010452+0000", 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.010452+0000", 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.010453+0000", 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1999999999999998e-07 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.010459+0000", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5919999999999997e-06 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.128848+0000", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.11838805299999999 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.128859+0000", 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1028e-05 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.129036+0000", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017784800000000001 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.129056+0000", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0027999999999999e-05 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.148453+0000", 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019396813999999998 2024-09-14T04:08:01.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:07:30.148493+0000", 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9789000000000001e-05 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:08:01.968 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:08:01.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:02.008 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:02.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:02.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:02.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:02.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:02.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:02.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:02.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:02.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:02.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:02.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:02.721 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:02 smithi001 podman[183083]: 2024-09-14 04:08:02.253797788 +0000 UTC m=+1.471543772 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:08:02.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:02.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:02.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:02.924 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:02.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:02.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:03.055 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:03.085 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:03.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:03.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:03.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:03.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:08:03.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:03.345 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:03.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:03.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:03.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:03 smithi012 ceph-mon[101180]: pgmap v604: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:03.638 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:03.654 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:03.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:03 smithi001 ceph-mon[112165]: pgmap v604: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:03 smithi001 ceph-mon[107399]: pgmap v604: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:03.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:03.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:03.906 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:03.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:03.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:04.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:04.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:04.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:04.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:04.216 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:04.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:04.336 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3404/41799 objects degraded (8.144%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:04.353 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:04.368 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3404/41799 objects degraded (8.144%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:04.369 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3404/41799 objects degraded (8.144%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:04.369 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:04 smithi001 podman[183083]: 2024-09-14 04:08:04.159552598 +0000 UTC m=+3.377298570 container create f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:08:04.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:04.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:04.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:08:04.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:04.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:04.667 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:04 smithi001 podman[183083]: 2024-09-14 04:08:04.481274051 +0000 UTC m=+3.699020045 container init f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231) 2024-09-14T04:08:04.667 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:04 smithi001 podman[183083]: 2024-09-14 04:08:04.489309793 +0000 UTC m=+3.707055773 container start f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T04:08:04.667 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:04 smithi001 bash[183083]: f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d 2024-09-14T04:08:04.667 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:04 smithi001 systemd[1]: Started Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:08:04.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:04.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:04.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:04.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:04.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:05.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:05.061 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:05.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:05.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:05.228 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:05.258 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:05 smithi001 ceph-mon[112165]: pgmap v605: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:05.258 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:05.258 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:05 smithi001 ceph-mon[107399]: pgmap v605: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:05.258 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:05.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:05 smithi012 ceph-mon[101180]: pgmap v605: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 870 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 3404/41799 objects degraded (8.144%); 2/41799 objects misplaced (0.005%) 2024-09-14T04:08:05.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:05.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:05.387 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-14T04:08:05.421 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:08:05.421 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:08:05.451 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:08:05.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:08:05.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:05.482 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:08:05.482 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:08:05.520 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:08:05.521 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:08:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:05.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:05.562 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:08:05.562 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:08:05.600 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:08:05.601 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:08:05.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:05.645 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:08:05.645 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:08:05.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:05.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:05.787 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:05.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:05.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:05.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:06.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:06.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:06.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:06.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:06.220 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:05 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:08:05.877+0000 7f1f455c6640 -1 Falling back to public interface 2024-09-14T04:08:06.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:06.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:06.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/891370810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:06.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:06.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:06.501 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/891370810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:06.502 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/891370810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:06.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:06.605 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:06.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:06.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:06.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:06.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:06.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:06.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:06] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-14T04:08:06.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:06.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:06.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:07.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:07.118 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:07.215 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:07.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:07.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:07.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:07.408 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:07 smithi001 ceph-mon[107399]: pgmap v606: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 774 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 3101/37200 objects degraded (8.336%); 2/37200 objects misplaced (0.005%) 2024-09-14T04:08:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:07 smithi001 ceph-mon[112165]: pgmap v606: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 774 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 3101/37200 objects degraded (8.336%); 2/37200 objects misplaced (0.005%) 2024-09-14T04:08:07.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:07.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:07.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:07 smithi012 ceph-mon[101180]: pgmap v606: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 774 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 3101/37200 objects degraded (8.336%); 2/37200 objects misplaced (0.005%) 2024-09-14T04:08:07.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:07.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:07.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:07.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:07.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:07.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:07.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:07.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:08.036 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:08.036 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:08.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:08.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:08.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:08.243 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:08.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:08.380 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:08.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:08.482 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:08.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:08.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:08.655 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:08.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:08.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:08.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:08:08.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:08.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:08.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:09.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:09.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:09.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:09.211 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:08:09.242 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:08:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.243 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:09.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:09.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:09.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:09.455 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:09 smithi001 ceph-mon[112165]: pgmap v607: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 556 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 2173/26754 objects degraded (8.122%); 2/26754 objects misplaced (0.007%) 2024-09-14T04:08:09.455 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2173/26754 objects degraded (8.122%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:09.458 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:09 smithi001 ceph-mon[107399]: pgmap v607: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 556 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 2173/26754 objects degraded (8.122%); 2/26754 objects misplaced (0.007%) 2024-09-14T04:08:09.458 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2173/26754 objects degraded (8.122%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:09.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:09.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:09 smithi012 ceph-mon[101180]: pgmap v607: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 556 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 2173/26754 objects degraded (8.122%); 2/26754 objects misplaced (0.007%) 2024-09-14T04:08:09.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2173/26754 objects degraded (8.122%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:09.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:09.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:09.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:09.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:09.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:09.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:09.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:09.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:09.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:10.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:10.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:10.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:10.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:10.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.368 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:10.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:10.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:10.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:10.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:10.641 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:10.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:10.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:10.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:10.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:11.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:11.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:11.185 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:11.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:11.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:08:11.276 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:08:11.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:11.430 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: pgmap v608: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 519 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2173/24975 objects degraded (8.701%); 2/24975 objects misplaced (0.008%) 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1361062948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:08:11.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:11.478 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:11.478 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:11.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.662 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:11.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: pgmap v608: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 519 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2173/24975 objects degraded (8.701%); 2/24975 objects misplaced (0.008%) 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1361062948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: pgmap v608: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 519 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2173/24975 objects degraded (8.701%); 2/24975 objects misplaced (0.008%) 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1361062948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:08:11.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:11.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:11.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:11.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:11.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:11.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:11.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:12.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.089 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:12.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:12.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:08:12.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:12.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:12.329 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:12.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:12.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:12.471 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:08:12.334+0000 7f1f455c6640 -1 osd.0 151 log_to_monitors true 2024-09-14T04:08:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[107399]: pgmap v609: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 397 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1623/19113 objects degraded (8.492%); 2/19113 objects misplaced (0.010%) 2024-09-14T04:08:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:08:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[107399]: osdmap e171: 8 total, 7 up, 8 in 2024-09-14T04:08:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[107399]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-14T04:08:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[112165]: pgmap v609: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 397 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1623/19113 objects degraded (8.492%); 2/19113 objects misplaced (0.010%) 2024-09-14T04:08:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:08:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[112165]: osdmap e171: 8 total, 7 up, 8 in 2024-09-14T04:08:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:12 smithi001 ceph-mon[112165]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-14T04:08:12.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:12.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:12.555 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:12.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:12 smithi012 ceph-mon[101180]: pgmap v609: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 397 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1623/19113 objects degraded (8.492%); 2/19113 objects misplaced (0.010%) 2024-09-14T04:08:12.598 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:12 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:12.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:08:12.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:12 smithi012 ceph-mon[101180]: osdmap e171: 8 total, 7 up, 8 in 2024-09-14T04:08:12.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:12 smithi012 ceph-mon[101180]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-14T04:08:12.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:12.656 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:12.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:12.758 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:12.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:12.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:12.830 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:12.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:12.954 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:12.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:13.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.093 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:13.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:13.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:13.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:13.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:13.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:13.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:13.500 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:13 smithi012 ceph-mon[101180]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.500 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:13 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:13.546 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:13 smithi001 ceph-mon[112165]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.546 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:13 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.546 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:13 smithi001 ceph-mon[107399]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.546 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:13 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:08:13.547 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:08:13 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:08:13.327+0000 7f1f3d370640 -1 osd.0 172 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:08:13.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:13.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:13.735 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:13.771 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:13.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:13.817 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:08:13.817 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:08:13.817 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:08:13.818 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:08:13.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:13.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:13.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:14.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:14.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:14.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:08:14.188 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:14.290 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:14.356 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:14.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:14.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:14.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: pgmap v611: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 274 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 1051/13185 objects degraded (7.971%); 2/13185 objects misplaced (0.015%) 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: osdmap e172: 8 total, 7 up, 8 in 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 1051/13185 objects degraded (7.971%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: pgmap v611: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 274 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 1051/13185 objects degraded (7.971%); 2/13185 objects misplaced (0.015%) 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: osdmap e172: 8 total, 7 up, 8 in 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 1051/13185 objects degraded (7.971%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:14.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:14.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: pgmap v611: 99 pgs: 12 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 274 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 1051/13185 objects degraded (7.971%); 2/13185 objects misplaced (0.015%) 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: osdmap e172: 8 total, 7 up, 8 in 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: from='osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:08:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 1051/13185 objects degraded (7.971%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:14.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:14.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:14.728 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:14.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:14.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:14.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:14.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:14.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:15.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:15.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:15.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:15.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:15.310 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:15.310 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:15.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:15.356 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:15.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:15.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:15.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:15.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:15 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:08:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:15 smithi012 ceph-mon[101180]: osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728] boot 2024-09-14T04:08:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:15 smithi012 ceph-mon[101180]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T04:08:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:15 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:08:15.605 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[112165]: osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728] boot 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[112165]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[107399]: osd.0 [v2:172.21.15.1:6802/3225184728,v1:172.21.15.1:6803/3225184728] boot 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[107399]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T04:08:15.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:15 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:08:15.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:15.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:15.834 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:15.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:15.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:15.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:15.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:16.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:16.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:16.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:16.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:08:16.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:16.220 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:16.228 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:08:16.228 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:08:16.236 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:16.247 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:08:16.247 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:08:16.268 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:08:16.269 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:08:16.285 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:08:16.285 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:08:16.301 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:08:16.301 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:08:16.332 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:08:16.333 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:08:16.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:16.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:08:16.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:08:16.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:16.382 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:08:16.382 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:08:16.433 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:16.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:16.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[107399]: pgmap v614: 99 pgs: 1 remapped+peering, 2 peering, 11 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 55 active+clean; 240 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 1051/11556 objects degraded (9.095%); 2/11556 objects misplaced (0.017%) 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[107399]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2250189406' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[112165]: pgmap v614: 99 pgs: 1 remapped+peering, 2 peering, 11 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 55 active+clean; 240 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 1051/11556 objects degraded (9.095%); 2/11556 objects misplaced (0.017%) 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[112165]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T04:08:16.498 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2250189406' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:16.558 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:16 smithi012 ceph-mon[101180]: pgmap v614: 99 pgs: 1 remapped+peering, 2 peering, 11 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 55 active+clean; 240 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 1051/11556 objects degraded (9.095%); 2/11556 objects misplaced (0.017%) 2024-09-14T04:08:16.558 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:16 smithi012 ceph-mon[101180]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T04:08:16.558 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2250189406' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:16.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:16.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:16.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:16.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:16.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:16.863 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:16] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-14T04:08:16.897 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:16.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:16.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:16.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:17.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:17.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:17.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:17.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:17.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.323 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:17.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:17.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:17.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:17.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[107399]: 1.0 scrub starts 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[107399]: 1.0 scrub ok 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[107399]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[107399]: 9.9 starting backfill to osd.0 from (140'2119,151'10044] MAX to 172'16254 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[112165]: 1.0 scrub starts 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[112165]: 1.0 scrub ok 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[112165]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T04:08:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:17 smithi001 ceph-mon[112165]: 9.9 starting backfill to osd.0 from (140'2119,151'10044] MAX to 172'16254 2024-09-14T04:08:17.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:17.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:17 smithi012 ceph-mon[101180]: 1.0 scrub starts 2024-09-14T04:08:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:17 smithi012 ceph-mon[101180]: 1.0 scrub ok 2024-09-14T04:08:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:17 smithi012 ceph-mon[101180]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T04:08:17.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:17 smithi012 ceph-mon[101180]: 9.9 starting backfill to osd.0 from (140'2119,151'10044] MAX to 172'16254 2024-09-14T04:08:17.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:17.738 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:17.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:17.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:17.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:17.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:18.015 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:18.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 131591 objects 2024-09-14T04:08:18.035 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :152.218 2024-09-14T04:08:18.117 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:18.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:18.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:18.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:18.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:18.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:18.408 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:18 smithi001 ceph-mon[107399]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 5 peering, 8 active+clean+remapped, 5 active+undersized, 3 active+undersized+degraded, 73 active+clean; 72 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 6391/3504 objects degraded (182.392%); 1/3504 objects misplaced (0.029%) 2024-09-14T04:08:18.409 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:18 smithi001 ceph-mon[112165]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 5 peering, 8 active+clean+remapped, 5 active+undersized, 3 active+undersized+degraded, 73 active+clean; 72 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 6391/3504 objects degraded (182.392%); 1/3504 objects misplaced (0.029%) 2024-09-14T04:08:18.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:18.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:18 smithi012 ceph-mon[101180]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 5 peering, 8 active+clean+remapped, 5 active+undersized, 3 active+undersized+degraded, 73 active+clean; 72 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 6391/3504 objects degraded (182.392%); 1/3504 objects misplaced (0.029%) 2024-09-14T04:08:18.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:18.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:18.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:18.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:18.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:18.889 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:18.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:18.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:19.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:19.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:19.140 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:19.206 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:08:19.206 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:08:19.206 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2024-09-14T04:08:19.206 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:08:19.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:08:19.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:19.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:19.329 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:08:19.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:19.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:19.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:19.523 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:19 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:08:19.523 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:19 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:08:19.523 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2405837873' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.523 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.523 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6391/3504 objects degraded (182.392%), 4 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:19.540 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:19.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:19.600 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:08:19.600 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:08:19.600 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-14T04:08:19.601 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:08:19.601 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:08:19.601 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2405837873' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6391/3504 objects degraded (182.392%), 4 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:19.625 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:08:19.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:08:19.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2405837873' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:19.626 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6391/3504 objects degraded (182.392%), 4 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:08:19.641 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:19.662 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-14T04:08:19.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-14T04:08:19.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:19.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:19.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:08:19.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:19.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:19.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:19.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:20.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:20.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:20.049 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:20.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:20.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:20.270 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:20.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:20.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:20.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:20.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 90 active+clean; 72 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6386/3504 objects degraded (182.249%) 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:20.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:20.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:20.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:20.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:20.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:20.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 90 active+clean; 72 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6386/3504 objects degraded (182.249%) 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 90 active+clean; 72 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6386/3504 objects degraded (182.249%) 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:20.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:20.792 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:20.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:20.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:20.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:21.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:21.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:21.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:21.167 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:21.222 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:21.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:21.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:21.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:21 smithi001 ceph-mon[107399]: 1.2 scrub starts 2024-09-14T04:08:21.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:21 smithi001 ceph-mon[107399]: 1.2 scrub ok 2024-09-14T04:08:21.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:21 smithi001 ceph-mon[112165]: 1.2 scrub starts 2024-09-14T04:08:21.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:21 smithi001 ceph-mon[112165]: 1.2 scrub ok 2024-09-14T04:08:21.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:21.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:21.500 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:21.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:21.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:21.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:21.548 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:21 smithi012 ceph-mon[101180]: 1.2 scrub starts 2024-09-14T04:08:21.548 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:21 smithi012 ceph-mon[101180]: 1.2 scrub ok 2024-09-14T04:08:21.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:21.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:21.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:21.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:21.784 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:08:21.784 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:08:21.784 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:08:21.784 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:08:21.785 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:08:21.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:21.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:21.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:21.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:21.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:22.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:22.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:22.147 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:22.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:22.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:22.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:22.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:22.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:22.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:22 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:08:22.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:22 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:08:22.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:22 smithi012 ceph-mon[101180]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 95 active+clean; 41 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6386/2019 objects degraded (316.295%) 2024-09-14T04:08:22.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:08:22.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:22.557 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:22.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:22.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:22.697 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[107399]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 95 active+clean; 41 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6386/2019 objects degraded (316.295%) 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:08:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:22 smithi001 ceph-mon[112165]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 95 active+clean; 41 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6386/2019 objects degraded (316.295%) 2024-09-14T04:08:22.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:22.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:22.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:22.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:22.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:23.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:23.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:23.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:08:23.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:23.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:23.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:23.247 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:23.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:23.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:23.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:23.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:23.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:23.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:23.587 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:23.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:23.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:23.712 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:23.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:23.838 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:23.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:23.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:23.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:24.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:24.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:24.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:24.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:24.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:24.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34299.0:194967 8.4 8:23d6bb2d:::benchmark_data_smithi001_125658_object69728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:00:06.258306+0000", 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 498.14469730000002, 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45464791999999998, 2024-09-14T04:08:24.409 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34299", 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 194967 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258306+0000", 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258306+0000", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258305+0000", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258310+0000", 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0549999999999999e-06 2024-09-14T04:08:24.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258320+0000", 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5240000000000006e-06 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258324+0000", 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6229999999999996e-06 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258368+0000", 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3217000000000003e-05 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258459+0000", 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1533000000000006e-05 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.258526+0000", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7267000000000005e-05 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.262836+0000", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043096389999999997 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.264347+0000", 2024-09-14T04:08:24.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015112669999999999 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.712862+0000", 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.44851502599999998 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.712916+0000", 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4039000000000002e-05 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:00:06.712954+0000", 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7719000000000003e-05 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166779 9.d e153/135 9:b228385c:::benchmark_data_smithi001_156540_object35185:head v 153'10400, mlcod=153'10398)", 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 114.20907343499999, 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18896571700000001, 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.416 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193929+0000", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193937+0000", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2699999999999999e-06 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193947+0000", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0465000000000001e-05 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193953+0000", 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3790000000000003e-06 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195833+0000", 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001880044 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195851+0000", 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7334999999999999e-05 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.230956+0000", 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.035105619999999997 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382896+0000", 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.151939308 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166789 9.5 9:a97ca575:::benchmark_data_smithi001_156540_object35195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:24.420 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 114.20194735600001, 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18248584100000001, 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166789 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:08:24.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201055+0000", 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201059+0000", 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.873e-06 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201072+0000", 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2414e-05 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201078+0000", 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3459999999999999e-06 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211075+0000", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0099971409999999993 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211180+0000", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010500799999999999 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 7", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211291+0000", 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111189 2024-09-14T04:08:24.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382943+0000", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17165192400000001 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383362+0000", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041839200000000002 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383402+0000", 2024-09-14T04:08:24.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0034999999999999e-05 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383542+0000", 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000140316 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166788 9.d e153/135 9:b0a806cb:::benchmark_data_smithi001_156540_object35194:head v 153'10401, mlcod=153'10398)", 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 114.196159212, 2024-09-14T04:08:24.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17608984799999999, 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206850+0000", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2469999999999997e-06 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206952+0000", 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010228199999999999 2024-09-14T04:08:24.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206958+0000", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3870000000000003e-06 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211515+0000", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00455749 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211535+0000", 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0063000000000001e-05 2024-09-14T04:08:24.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382904+0000", 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.171368561 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382934+0000", 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.029e-05 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166805 9.7 9:e1bea35c:::benchmark_data_smithi001_156540_object35211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 114.175977579, 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16668448999999999, 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166805 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227025+0000", 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227030+0000", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7200000000000003e-06 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227044+0000", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3312999999999999e-05 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227051+0000", 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.267e-06 2024-09-14T04:08:24.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227101+0000", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9675000000000001e-05 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227226+0000", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012531400000000001 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227357+0000", 2024-09-14T04:08:24.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130882 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381943+0000", 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15458603400000001 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382052+0000", 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108744 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393514+0000", 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011461987999999999 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393546+0000", 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2799000000000002e-05 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393590+0000", 2024-09-14T04:08:24.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3458000000000003e-05 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393710+0000", 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120443 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166810 9.7 9:e438b2a3:::benchmark_data_smithi001_156540_object35216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:08:24.438 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 114.169834111, 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17089170000000001, 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166810 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233168+0000", 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233175+0000", 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.844e-06 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381981+0000", 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14880580400000001 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381992+0000", 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1167e-05 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382171+0000", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017918400000000001 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382318+0000", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014668999999999999 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382468+0000", 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015070599999999999 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393751+0000", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011282654 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.394106+0000", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035462900000000002 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.394146+0000", 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0611000000000001e-05 2024-09-14T04:08:24.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.403911+0000", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009764633 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.403957+0000", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5862000000000002e-05 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.404061+0000", 2024-09-14T04:08:24.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104258 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178516 9.7 9:e7cca128:::benchmark_data_smithi001_156540_object46922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.090086323999998, 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0890654740000001, 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.445 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178516 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312919+0000", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7429999999999998e-06 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312926+0000", 2024-09-14T04:08:24.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6290000000000004e-06 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312931+0000", 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.13e-06 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313179+0000", 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024747799999999998 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313398+0000", 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219287 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313540+0000", 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014250299999999999 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318169+0000", 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046281289999999999 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322748+0000", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004579751 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348314+0000", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025566169 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348324+0000", 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0469999999999999e-06 2024-09-14T04:08:24.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401919+0000", 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.053594996 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401983+0000", 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4016999999999994e-05 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178520 9.8 9:1c2e04c6:::benchmark_data_smithi001_156540_object46926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.083184528000004, 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0110979040000001, 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:24.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:24.453 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178520 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319821+0000", 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1720000000000001e-06 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319826+0000", 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7339999999999996e-06 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319831+0000", 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2799999999999997e-06 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320563+0000", 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073229899999999995 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320671+0000", 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107984 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320794+0000", 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122669 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.332330+0000", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011536103000000001 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343578+0000", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011247823000000001 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343588+0000", 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9939999999999995e-06 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330904+0000", 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98731630800000003 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330917+0000", 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2883e-05 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178521 9.7 9:e7156783:::benchmark_data_smithi001_156540_object46927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.083164523999997, 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069582149999999, 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178521 2024-09-14T04:08:24.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319838+0000", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319841+0000", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.086e-06 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319846+0000", 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6110000000000001e-06 2024-09-14T04:08:24.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319850+0000", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.211e-06 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322773+0000", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029226109999999999 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322906+0000", 2024-09-14T04:08:24.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013334800000000001 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323044+0000", 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137897 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350098+0000", 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027053395000000001 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350106+0000", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5750000000000001e-06 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416318+0000", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.066211786 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416396+0000", 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8298000000000001e-05 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426758+0000", 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010361354 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426797+0000", 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9487999999999999e-05 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178528 9.f 9:f3e5f4f3:::benchmark_data_smithi001_156540_object46934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.074658024000001, 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.087887475, 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178528 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328344+0000", 2024-09-14T04:08:24.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328349+0000", 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0950000000000003e-06 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328361+0000", 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1915e-05 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328367+0000", 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6549999999999998e-06 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350130+0000", 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021762786999999999 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350137+0000", 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8630000000000002e-06 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368572+0000", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.018434893 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368729+0000", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015681500000000001 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368871+0000", 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141779 2024-09-14T04:08:24.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402186+0000", 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033314990000000003 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402204+0000", 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8601e-05 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416128+0000", 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013924158000000001 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416185+0000", 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6869e-05 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416233+0000", 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7602999999999998e-05 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.473 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178532 9.7 9:e0e67f4c:::benchmark_data_smithi001_156540_object46938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.067713918999999, 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.088792084, 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178532 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.474 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335288+0000", 2024-09-14T04:08:24.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335293+0000", 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2470000000000002e-06 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335303+0000", 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0829000000000001e-05 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335310+0000", 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9009999999999997e-06 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350158+0000", 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014848175999999999 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350165+0000", 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416444+0000", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0662793960000001 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416514+0000", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9778999999999997e-05 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416588+0000", 2024-09-14T04:08:24.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3963999999999999e-05 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423845+0000", 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072567669999999999 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423979+0000", 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013418300000000001 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423990+0000", 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1459e-05 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424029+0000", 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.892e-05 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424082+0000", 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2386000000000003e-05 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.480 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178537 9.8 9.6a0d3bf8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.051931084000003, 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97989574099999999, 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178537 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.481 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351075+0000", 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.666e-06 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351081+0000", 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4840000000000001e-06 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351086+0000", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0699999999999997e-06 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351541+0000", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045494699999999998 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351549+0000", 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.498e-06 2024-09-14T04:08:24.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330958+0000", 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97940985199999997 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330968+0000", 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6840000000000008e-06 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178547 9.8 9.c42478a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.023924786999999, 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.952303813, 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178547 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379078+0000", 2024-09-14T04:08:24.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379083+0000", 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0189999999999997e-06 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379094+0000", 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1126e-05 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379099+0000", 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379137+0000", 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.786e-05 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0891e-05 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331371+0000", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95222277899999996 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331382+0000", 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1686e-05 2024-09-14T04:08:24.490 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178548 9.f 9:f6c46334:::benchmark_data_smithi001_156540_object46954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 99.023854841000002, 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0510430099999999, 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.491 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178548 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379152+0000", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0289999999999999e-06 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379161+0000", 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8910000000000001e-06 2024-09-14T04:08:24.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379166+0000", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1440000000000002e-06 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379197+0000", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0839000000000001e-05 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379202+0000", 2024-09-14T04:08:24.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1309999999999997e-06 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420298+0000", 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410965350000001 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420384+0000", 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5554000000000001e-05 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:08:24.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420466+0000", 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2263999999999997e-05 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426757+0000", 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006290459 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430060+0000", 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033035619999999999 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430121+0000", 2024-09-14T04:08:24.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1101999999999999e-05 2024-09-14T04:08:24.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:24 smithi012 ceph-mon[101180]: pgmap v621: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 49 objects/s recovering 2024-09-14T04:08:24.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430165+0000", 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4026000000000002e-05 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430192+0000", 2024-09-14T04:08:24.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6142000000000001e-05 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262060 9.5 9:aa922bae:::benchmark_data_smithi001_156540_object130466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 7.9197683080000001, 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.199390137, 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262060 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483234+0000", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483238+0000", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483247+0000", 2024-09-14T04:08:24.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8869999999999998e-06 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483252+0000", 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.882e-06 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483300+0000", 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7879e-05 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483306+0000", 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7139999999999998e-06 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647214+0000", 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16390846000000001 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647330+0000", 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011582 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647443+0000", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011323999999999999 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653349+0000", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059055039999999998 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653931+0000", 2024-09-14T04:08:24.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058244600000000003 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682380+0000", 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028448703999999998 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682433+0000", 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3291000000000002e-05 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682625+0000", 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019192999999999999 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262062 9.b 9:db855ab8:::benchmark_data_smithi001_156540_object130468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 7.9098692010000002, 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26506028100000001, 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.506 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262062 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:08:24.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493133+0000", 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493140+0000", 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7069999999999992e-06 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493155+0000", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4440999999999999e-05 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493162+0000", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6499999999999999e-06 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493204+0000", 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2209000000000002e-05 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493212+0000", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9929999999999995e-06 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654542+0000", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.161330167 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654750+0000", 2024-09-14T04:08:24.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020799300000000001 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654877+0000", 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012702300000000001 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663336+0000", 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084589330000000001 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663377+0000", 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1032999999999998e-05 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757657+0000", 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094279643999999996 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757711+0000", 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4691000000000002e-05 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758194+0000", 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048327800000000002 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262069 9.b 9:d49cb1c8:::benchmark_data_smithi001_156540_object130475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.639222+0000", 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 7.7637815010000004, 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.119548395, 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.513 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262069 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639222+0000", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639222+0000", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639221+0000", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639226+0000", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.237e-06 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639246+0000", 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9389e-05 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.639252+0000", 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6819999999999997e-06 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.658738+0000", 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019485797999999999 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.659238+0000", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000499662 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.659369+0000", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013095899999999999 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663421+0000", 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040520290000000004 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.664050+0000", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062915099999999999 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758356+0000", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094306272999999996 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758435+0000", 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8910000000000002e-05 2024-09-14T04:08:24.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758770+0000", 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00033521000000000001 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262924 9.5 9:a7ddf137:::benchmark_data_smithi001_156540_object131330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:17.689896+0000", 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.7131071200000001, 2024-09-14T04:08:24.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13567175000000001, 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262924 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689896+0000", 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689896+0000", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689896+0000", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689899+0000", 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6389999999999998e-06 2024-09-14T04:08:24.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689907+0000", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9149999999999996e-06 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689911+0000", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8539999999999999e-06 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.689954+0000", 2024-09-14T04:08:24.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2871999999999999e-05 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.690078+0000", 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012423999999999999 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.690187+0000", 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109078 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.694847+0000", 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046593600000000004 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.695261+0000", 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041463699999999998 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825026+0000", 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12976501700000001 2024-09-14T04:08:24.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825086+0000", 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9382999999999997e-05 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825568+0000", 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048235699999999999 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262933 9.5 9:ad3356a2:::benchmark_data_smithi001_156540_object131339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:17.703240+0000", 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.6997631890000005, 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12261046, 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262933 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.526 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703240+0000", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703240+0000", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703240+0000", 2024-09-14T04:08:24.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703243+0000", 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.799e-06 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703251+0000", 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.25e-06 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703255+0000", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0690000000000003e-06 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703288+0000", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3231000000000001e-05 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703374+0000", 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5804999999999996e-05 2024-09-14T04:08:24.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.703440+0000", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6561000000000002e-05 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.708110+0000", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046702389999999996 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.708805+0000", 2024-09-14T04:08:24.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069503900000000005 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825616+0000", 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11681050599999999 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825656+0000", 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0244000000000003e-05 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.825851+0000", 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000194427 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262960 9.5 9:a27e5265:::benchmark_data_smithi001_156540_object131366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:17.737603+0000", 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.6654004449999995, 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11705939899999999, 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:24.532 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262960 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737603+0000", 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737603+0000", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737603+0000", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737606+0000", 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0340000000000001e-06 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737613+0000", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.182e-06 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.737618+0000", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3249999999999998e-06 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.738732+0000", 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011140810000000001 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.738839+0000", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010640400000000001 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.738902+0000", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3885000000000002e-05 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.743609+0000", 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047068099999999996 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.743790+0000", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018052899999999999 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.854236+0000", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.110446452 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.854275+0000", 2024-09-14T04:08:24.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8912000000000003e-05 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:17.854662+0000", 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038719600000000002 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:24.538 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:08:24.539 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:24 smithi001 ceph-mon[112165]: pgmap v621: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 49 objects/s recovering 2024-09-14T04:08:24.539 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:24.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:24 smithi001 ceph-mon[107399]: pgmap v621: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 49 objects/s recovering 2024-09-14T04:08:24.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:24.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:24.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:24.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:24.786 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:24.810 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:24.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:24.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:24.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:24.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:25.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.065 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:25.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:25.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:25.193 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:25.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:25.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:25.348 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_4' removed 2024-09-14T04:08:25.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:08:25.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:25.450 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:08:25.451 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:08:25.470 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:08:25.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:08:25.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:25.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:25.486 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:08:25.486 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:08:25.505 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:08:25.505 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:08:25.521 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:08:25.521 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:08:25.538 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:08:25.538 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:08:25.555 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:08:25.555 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:08:25.571 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:08:25.571 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:08:25.571 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1488923239' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.571 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:25 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.572 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1488923239' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.572 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:25 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1488923239' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:25 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:08:25.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:25.618 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:25.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:25.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:25.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:25.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:25.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:25.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:25.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:25.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:26.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.123 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:26.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:26.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:26.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:26.372 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:26.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:26.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:26.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:26 smithi012 ceph-mon[101180]: pgmap v622: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 43 objects/s recovering 2024-09-14T04:08:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:26 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:08:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:26 smithi012 ceph-mon[101180]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T04:08:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2110172732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:26.599 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:26.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:26.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[107399]: pgmap v622: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 43 objects/s recovering 2024-09-14T04:08:26.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[107399]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2110172732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[112165]: pgmap v622: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 681 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 12503/78 objects degraded (16029.487%); 0 B/s, 43 objects/s recovering 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[112165]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T04:08:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2110172732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:26.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:26.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:26.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:26.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:26] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-14T04:08:26.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:26.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:26.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:27.098 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:27.125 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:27.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:27.250 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:27.265 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:27.265 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:27.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:27.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:27.388 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:08:27.388 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:08:27.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.503 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:27.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:27.627 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:27.633 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:08:27.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:27.752 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:27.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:27.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:27.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:27.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:27.934 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:08:27.935 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:08:27.935 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-14T04:08:27.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-14T04:08:27.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:27.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:08:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:28.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:28.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:28.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:28.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:28.174 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:08:28.174 INFO:teuthology.orchestra.run.smithi001.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:08:28.174 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:08:28.174 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:28.203 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:28.229 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:28.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:28.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:28.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:08:28.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: 1.7 scrub starts 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: 1.7 scrub ok 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: pgmap v624: 83 pgs: 83 active+clean; 629 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded) 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:08:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[107399]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: 1.7 scrub starts 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: 1.7 scrub ok 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: pgmap v624: 83 pgs: 83 active+clean; 629 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded) 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:08:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:28 smithi001 ceph-mon[112165]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T04:08:28.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:28.499 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:28.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: 1.7 scrub starts 2024-09-14T04:08:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: 1.7 scrub ok 2024-09-14T04:08:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: pgmap v624: 83 pgs: 83 active+clean; 629 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12503/78 objects degraded (16029.487%), 2 pgs degraded) 2024-09-14T04:08:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:08:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:08:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:28 smithi012 ceph-mon[101180]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T04:08:28.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:28.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:28.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:28.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:28.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:28.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:28.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:29.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:29.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:29.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:29.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:29.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:29.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:29.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:29.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:29.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:29.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:29.516 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:29 smithi012 ceph-mon[101180]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T04:08:29.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:29 smithi012 ceph-mon[101180]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T04:08:29.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:29.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:29.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:29 smithi001 ceph-mon[107399]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T04:08:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:29 smithi001 ceph-mon[107399]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T04:08:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:29 smithi001 ceph-mon[112165]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T04:08:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:29 smithi001 ceph-mon[112165]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T04:08:29.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:29.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:29.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:29.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:29.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:29.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:30.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:30.031 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:30.031 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:30.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:30.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:30.175 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:30.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:30.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:30.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:30.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:30 smithi001 ceph-mon[112165]: pgmap v627: 83 pgs: 83 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:30.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:30 smithi001 ceph-mon[107399]: pgmap v627: 83 pgs: 83 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:30.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:30.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:30.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:30 smithi012 ceph-mon[101180]: pgmap v627: 83 pgs: 83 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:30.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:30.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:08:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:30.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:30.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:30.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:30.878 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:30.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:30.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:31.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:31.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:31.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:31.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:31.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:31.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:31.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:31.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:31.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:31 smithi001 ceph-mon[107399]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T04:08:31.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:31 smithi001 ceph-mon[112165]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T04:08:31.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:31.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:31.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:31.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:31.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:31.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:31.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:31 smithi012 ceph-mon[101180]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T04:08:31.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:31.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:31.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:32.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:32.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:32.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:32.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:32.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:32.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:32.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:32.563 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:32 smithi012 ceph-mon[101180]: pgmap v630: 83 pgs: 5 peering, 78 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:32.563 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:32 smithi012 ceph-mon[101180]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T04:08:32.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:32.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:32.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:32 smithi001 ceph-mon[107399]: pgmap v630: 83 pgs: 5 peering, 78 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:32 smithi001 ceph-mon[107399]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T04:08:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:32 smithi001 ceph-mon[112165]: pgmap v630: 83 pgs: 5 peering, 78 active+clean; 629 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:32 smithi001 ceph-mon[112165]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T04:08:32.795 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:32.795 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:32.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:32.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:32.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:32.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:33.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:33.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:33.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:33.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:33.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:33.350 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:33.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:33.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:33.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:33 smithi012 ceph-mon[101180]: 1.4 scrub starts 2024-09-14T04:08:33.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:33 smithi012 ceph-mon[101180]: 1.4 scrub ok 2024-09-14T04:08:33.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:33 smithi012 ceph-mon[101180]: 1.3 scrub starts 2024-09-14T04:08:33.520 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:33 smithi012 ceph-mon[101180]: 1.3 scrub ok 2024-09-14T04:08:33.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:08:33.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:33.607 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:33.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[107399]: 1.4 scrub starts 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[107399]: 1.4 scrub ok 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[107399]: 1.3 scrub starts 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[107399]: 1.3 scrub ok 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[112165]: 1.4 scrub starts 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[112165]: 1.4 scrub ok 2024-09-14T04:08:33.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[112165]: 1.3 scrub starts 2024-09-14T04:08:33.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:33 smithi001 ceph-mon[112165]: 1.3 scrub ok 2024-09-14T04:08:33.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:33.746 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:33.775 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:33.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:33.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:33.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:33.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:33.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:33.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:34.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:34.109 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:34.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:34.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:34.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:34.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:34.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:34.433 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:08:34.433 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:08:34.433 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:08:34.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-09-14T04:08:34.459 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:34.502 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:34.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:34.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[107399]: 1.9 scrub starts 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[107399]: 1.9 scrub ok 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[107399]: pgmap v632: 83 pgs: 8 peering, 75 active+clean; 477 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 92 KiB/s wr, 20 op/s; 188 KiB/s, 0 objects/s recovering 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[107399]: 1.1 scrub starts 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[107399]: 1.1 scrub ok 2024-09-14T04:08:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[112165]: 1.9 scrub starts 2024-09-14T04:08:34.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[112165]: 1.9 scrub ok 2024-09-14T04:08:34.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[112165]: pgmap v632: 83 pgs: 8 peering, 75 active+clean; 477 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 92 KiB/s wr, 20 op/s; 188 KiB/s, 0 objects/s recovering 2024-09-14T04:08:34.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[112165]: 1.1 scrub starts 2024-09-14T04:08:34.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:34 smithi001 ceph-mon[112165]: 1.1 scrub ok 2024-09-14T04:08:34.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:34.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:34 smithi012 ceph-mon[101180]: 1.9 scrub starts 2024-09-14T04:08:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:34 smithi012 ceph-mon[101180]: 1.9 scrub ok 2024-09-14T04:08:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:34 smithi012 ceph-mon[101180]: pgmap v632: 83 pgs: 8 peering, 75 active+clean; 477 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 92 KiB/s wr, 20 op/s; 188 KiB/s, 0 objects/s recovering 2024-09-14T04:08:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:34 smithi012 ceph-mon[101180]: 1.1 scrub starts 2024-09-14T04:08:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:34 smithi012 ceph-mon[101180]: 1.1 scrub ok 2024-09-14T04:08:34.853 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:34.875 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:34.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:34.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:35.000 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:35.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:35.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:35.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:35.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:35.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:35.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:35.351 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:35.351 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:35.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:35.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:35.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:35.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:35.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:35.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:35.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: 1.5 scrub starts 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: 1.5 scrub ok 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: 1.6 scrub starts 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: 1.6 scrub ok 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[107399]: pgmap v633: 83 pgs: 8 peering, 75 active+clean; 425 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 36 KiB/s wr, 6 op/s; 122 KiB/s, 0 objects/s recovering 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: 1.5 scrub starts 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: 1.5 scrub ok 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: 1.6 scrub starts 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: 1.6 scrub ok 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:35 smithi001 ceph-mon[112165]: pgmap v633: 83 pgs: 8 peering, 75 active+clean; 425 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 36 KiB/s wr, 6 op/s; 122 KiB/s, 0 objects/s recovering 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: 1.5 scrub starts 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: 1.5 scrub ok 2024-09-14T04:08:35.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: 1.6 scrub starts 2024-09-14T04:08:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: 1.6 scrub ok 2024-09-14T04:08:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:35.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:35 smithi012 ceph-mon[101180]: pgmap v633: 83 pgs: 8 peering, 75 active+clean; 425 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 36 KiB/s wr, 6 op/s; 122 KiB/s, 0 objects/s recovering 2024-09-14T04:08:35.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:35.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:35.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:35.881 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:35.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:35.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:36.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:36.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:08:36.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:36.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:36.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:36.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:36.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:36.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:36.287 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:36.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:36.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:36 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:08:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:36 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:08:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:36 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:08:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:36 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:08:36.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:36.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:36.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:36 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:08:36.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:36 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:08:36.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.642 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:36.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:36.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:36.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:36.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:36.891 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:36] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.51.0" 2024-09-14T04:08:36.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:36.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:36.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:37.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:37.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:37.073 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:37.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:37.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:37.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:37.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:37.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:37.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[107399]: 1.7 deep-scrub starts 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[107399]: 1.7 deep-scrub ok 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[107399]: pgmap v634: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[112165]: 1.7 deep-scrub starts 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[112165]: 1.7 deep-scrub ok 2024-09-14T04:08:37.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:37 smithi001 ceph-mon[112165]: pgmap v634: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:37.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:37.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:37.617 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:37.640 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:37.712 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:37 smithi012 ceph-mon[101180]: 1.7 deep-scrub starts 2024-09-14T04:08:37.712 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:37 smithi012 ceph-mon[101180]: 1.7 deep-scrub ok 2024-09-14T04:08:37.712 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:37 smithi012 ceph-mon[101180]: pgmap v634: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:37.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:37.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:37.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:37.891 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:37.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:37.971 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:37.971 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:37.993 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:38.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:38.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:38.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:38.141 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:38.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:38.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:38.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:38.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:38.496 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:38.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:38.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:08:38.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:38.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:38.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:38.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:38.791 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:08:38.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:38.898 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:38.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:39.000 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:39.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:39.043 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:08:39.066 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:08:39.066 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:08:39.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:39.149 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:39.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:39.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:08:39.379 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:39.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:39.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:39.531 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:39.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:39.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:39.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:39.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:39.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:39.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:39.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:40.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:40.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:08:40.122 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:08:40.123 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:08:40.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:40.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[107399]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[107399]: pgmap v636: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2788802830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:40.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:08:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[112165]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T04:08:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[112165]: pgmap v636: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:40.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2788802830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:40.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:40.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:40.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:40 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:40.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/768911579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:08:40.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:40 smithi012 ceph-mon[101180]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T04:08:40.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:40 smithi012 ceph-mon[101180]: pgmap v636: 83 pgs: 83 active+clean; 687 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:40.309 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2788802830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:40.354 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:40.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:40.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:40.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:40.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:40.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:40.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:40.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:40.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:40.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:40.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:40.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:41.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:41.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:41.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:41.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:41.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:41.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:41.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:41.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:41.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:162440 9.9 9:9a8f3dba:::benchmark_data_smithi001_156540_object30846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:24.513539+0000", 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 136.997525201, 2024-09-14T04:08:41.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12569089, 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 162440 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513539+0000", 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513539+0000", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513538+0000", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513543+0000", 2024-09-14T04:08:41.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.955e-06 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513553+0000", 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0458999999999999e-05 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.513560+0000", 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9450000000000004e-06 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.538093+0000", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024533261000000001 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.538240+0000", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014618500000000001 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.538353+0000", 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011341300000000001 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.553636+0000", 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015283246 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.639056+0000", 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.085419991000000001 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.639141+0000", 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5088999999999998e-05 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:24.639230+0000", 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8432999999999998e-05 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:162977 9.3 9:c2d413a3:::benchmark_data_smithi001_156540_object31383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 136.31093811900001, 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14203296300000001, 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 162977 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.524 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200124+0000", 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200130+0000", 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0850000000000002e-06 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200144+0000", 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3104e-05 2024-09-14T04:08:41.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200151+0000", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2250000000000003e-06 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201100+0000", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094949499999999996 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201218+0000", 2024-09-14T04:08:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117963 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201348+0000", 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012961299999999999 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206267+0000", 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049196209999999999 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206296+0000", 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8626e-05 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212462+0000", 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006165472 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.341995+0000", 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12953315100000001 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.342049+0000", 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3891999999999998e-05 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.342159+0000", 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110272 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:162980 9.3 9:cb003418:::benchmark_data_smithi001_156540_object31386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.530 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 136.30621214000001, 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14076661400000001, 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 162980 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204850+0000", 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204856+0000", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.84e-06 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204867+0000", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0231000000000001e-05 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204873+0000", 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7889999999999997e-06 2024-09-14T04:08:41.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206317+0000", 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014438579999999999 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206431+0000", 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011421000000000001 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:08:41.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206554+0000", 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012220300000000001 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212466+0000", 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059127829999999996 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212489+0000", 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2138999999999999e-05 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.213175+0000", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00068672299999999998 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.344992+0000", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13181647899999999 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.345505+0000", 2024-09-14T04:08:41.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051285999999999997 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.345618+0000", 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011390100000000001 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166779 9.d 9:b228385c:::benchmark_data_smithi001_156540_object35185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.320031594, 2024-09-14T04:08:41.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19259300700000001, 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166779 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191031+0000", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191036+0000", 2024-09-14T04:08:41.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6450000000000004e-06 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191046+0000", 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6949999999999995e-06 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191052+0000", 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2009999999999996e-06 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191095+0000", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3046000000000001e-05 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191200+0000", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105394 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191304+0000", 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104038 2024-09-14T04:08:41.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.197245+0000", 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059409659999999998 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.208912+0000", 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011666434999999999 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383544+0000", 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17463272099999999 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383590+0000", 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5046999999999997e-05 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383625+0000", 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5918999999999999e-05 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166788 9.d 9:b0a806cb:::benchmark_data_smithi001_156540_object35194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.31078169400001, 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18345835699999999, 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166788 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.544 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200287+0000", 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0119999999999996e-06 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200301+0000", 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4596999999999999e-05 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200308+0000", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7349999999999999e-06 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201962+0000", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001654483 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.203690+0000", 2024-09-14T04:08:41.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001727803 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.203796+0000", 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105812 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.210818+0000", 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070218889999999999 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.210946+0000", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012832300000000001 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383661+0000", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17271468100000001 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383708+0000", 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7357000000000001e-05 2024-09-14T04:08:41.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383741+0000", 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2447999999999998e-05 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166802 9.d 9:be97eccd:::benchmark_data_smithi001_156540_object35208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.29410094599999, 2024-09-14T04:08:41.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.166921759, 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166802 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216965+0000", 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5220000000000002e-06 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216971+0000", 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7679999999999997e-06 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216975+0000", 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4889999999999997e-06 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217020+0000", 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4070000000000003e-05 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217127+0000", 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010763400000000001 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217236+0000", 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108507 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.229705+0000", 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012468918000000001 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.234522+0000", 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048169500000000004 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383773+0000", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14925143099999999 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383850+0000", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6933000000000004e-05 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383885+0000", 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4947999999999998e-05 2024-09-14T04:08:41.556 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166799 9.1 e153/149 9:8c8a635f:::benchmark_data_smithi001_156540_object35205:head v 153'10484, mlcod=153'10481)", 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.28934159400001, 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.159769142, 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221728+0000", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1840000000000004e-06 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221738+0000", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.027e-05 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221745+0000", 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2920000000000001e-06 2024-09-14T04:08:41.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.226758+0000", 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050130519999999996 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.226786+0000", 2024-09-14T04:08:41.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8493999999999999e-05 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381450+0000", 2024-09-14T04:08:41.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15466439100000001 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381492+0000", 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1084000000000002e-05 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166808 9.1 e153/149 9:85c01d48:::benchmark_data_smithi001_156540_object35214:head v 153'10485, mlcod=153'10481)", 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:08:41.563 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.27885311099999, 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.149302823, 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:08:41.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232210+0000", 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232217+0000", 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2280000000000001e-06 2024-09-14T04:08:41.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232235+0000", 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7819e-05 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232242+0000", 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6089999999999997e-06 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232340+0000", 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8549999999999997e-05 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232361+0000", 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0945999999999999e-05 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381494+0000", 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14913268199999999 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381514+0000", 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9678999999999999e-05 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166805 9.7 e153/149 9:e1bea35c:::benchmark_data_smithi001_156540_object35211:head v 153'10387, mlcod=153'10385)", 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.27809002199999, 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14852269200000001, 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.568 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232977+0000", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.551e-06 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232983+0000", 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9780000000000002e-06 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232987+0000", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.997e-06 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233201+0000", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021345899999999999 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233225+0000", 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4219e-05 2024-09-14T04:08:41.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381381+0000", 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14815647600000001 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381497+0000", 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115259 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166809 9.2 e153/149 9:4e12f0fd:::benchmark_data_smithi001_156540_object35215:head v 153'10294, mlcod=153'10291)", 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.275109867, 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15257738200000001, 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235953+0000", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235960+0000", 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5910000000000001e-06 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235971+0000", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0769e-05 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235977+0000", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2639999999999997e-06 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.236047+0000", 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9840999999999998e-05 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381832+0000", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14578494 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.388490+0000", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066577950000000002 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.388532+0000", 2024-09-14T04:08:41.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2002000000000002e-05 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:167270 9.e 9:78c90245:::benchmark_data_smithi001_156540_object35676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.975750+0000", 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 130.53531403100001, 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12376390299999999, 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 167270 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975750+0000", 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975750+0000", 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975749+0000", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975754+0000", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0189999999999997e-06 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975763+0000", 2024-09-14T04:08:41.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8470000000000008e-06 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975770+0000", 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5119999999999997e-06 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975896+0000", 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012618199999999999 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.976014+0000", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011792 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.976132+0000", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011766599999999999 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.988127+0000", 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011995469999999999 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099290+0000", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.111162847 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099428+0000", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013754699999999999 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099476+0000", 2024-09-14T04:08:41.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8733000000000002e-05 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099514+0000", 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7518999999999998e-05 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:167274 9.e 9:7aab58b7:::benchmark_data_smithi001_156540_object35680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.975803+0000", 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 130.53526057900001, 2024-09-14T04:08:41.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.1238601, 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 167274 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975803+0000", 2024-09-14T04:08:41.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975803+0000", 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975803+0000", 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975805+0000", 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2230000000000001e-06 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975812+0000", 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1759999999999998e-06 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.975816+0000", 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6920000000000001e-06 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.984026+0000", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082094479999999994 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.984133+0000", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106987 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.984241+0000", 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108399 2024-09-14T04:08:41.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.996759+0000", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012517967 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099333+0000", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102574155 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099587+0000", 2024-09-14T04:08:41.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00025390299999999998 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099626+0000", 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8763000000000001e-05 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:31.099664+0000", 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7688999999999998e-05 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:168104 9.d 9:befa1733:::benchmark_data_smithi001_156540_object36510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:32.086967+0000", 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 129.424096776, 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12587226700000001, 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 168104 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086967+0000", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086967+0000", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086966+0000", 2024-09-14T04:08:41.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086972+0000", 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1940000000000003e-06 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086987+0000", 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4975e-05 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.086997+0000", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4069999999999999e-06 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.087036+0000", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9376999999999999e-05 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.087135+0000", 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8666999999999996e-05 2024-09-14T04:08:41.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.087237+0000", 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102744 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.091063+0000", 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003825579 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.094861+0000", 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037975980000000001 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.212762+0000", 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.117901247 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.212806+0000", 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4280999999999998e-05 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:32.212840+0000", 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3402000000000003e-05 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:169629 9.d 9:ba255606:::benchmark_data_smithi001_156540_object38035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:34.138769+0000", 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 127.372294586, 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.126234235, 2024-09-14T04:08:41.596 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 169629 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138769+0000", 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138769+0000", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138768+0000", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138773+0000", 2024-09-14T04:08:41.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8099999999999997e-06 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138782+0000", 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4769999999999992e-06 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.138788+0000", 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9030000000000001e-06 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.140178+0000", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013897779999999999 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.140300+0000", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012222200000000001 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.140407+0000", 2024-09-14T04:08:41.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107125 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.144462+0000", 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040549690000000003 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.145821+0000", 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001358358 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.264921+0000", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.119100436 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.264965+0000", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3943000000000003e-05 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:34.265004+0000", 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8553999999999997e-05 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.602 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178529 9.3 9:c63b081d:::benchmark_data_smithi001_156540_object46935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 116.177045839, 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1003067309999999, 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178529 2024-09-14T04:08:41.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334017+0000", 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334021+0000", 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0679999999999998e-06 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334030+0000", 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4860000000000003e-06 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334037+0000", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.939e-06 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334160+0000", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012251300000000001 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334282+0000", 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012243599999999999 2024-09-14T04:08:41.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334405+0000", 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012238400000000001 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356132+0000", 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021727265999999999 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356140+0000", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4810000000000006e-06 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332059+0000", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97591887700000002 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332074+0000", 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5068999999999999e-05 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404414+0000", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.072339559 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404564+0000", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014992900000000001 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.434245+0000", 2024-09-14T04:08:41.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029680840999999999 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.434325+0000", 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0191000000000005e-05 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178530 9.3 9:ce32ae9f:::benchmark_data_smithi001_156540_object46936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:08:41.610 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 116.17702004, 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1012480839999998, 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178530 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334046+0000", 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1660000000000001e-06 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334051+0000", 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4229999999999996e-06 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334056+0000", 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6410000000000001e-06 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340669+0000", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066129869999999999 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340821+0000", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152435 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340975+0000", 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015369299999999999 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356177+0000", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015202126999999999 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356184+0000", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4230000000000002e-06 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332331+0000", 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.976147133 2024-09-14T04:08:41.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332341+0000", 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0288999999999999e-05 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404844+0000", 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0725033310000001 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:08:41.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404952+0000", 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108185 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.435228+0000", 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030275077000000001 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.435292+0000", 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4493999999999996e-05 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.617 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178555 9.3 9:c397c393:::benchmark_data_smithi001_156540_object46961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 115.08229219099999, 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0051882679999999, 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178555 2024-09-14T04:08:41.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428770+0000", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428777+0000", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1589999999999998e-06 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428789+0000", 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2266000000000001e-05 2024-09-14T04:08:41.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428795+0000", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9399999999999999e-06 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428845+0000", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9852999999999999e-05 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428854+0000", 2024-09-14T04:08:41.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6830000000000003e-06 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405044+0000", 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97618959400000005 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405148+0000", 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010432799999999999 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405258+0000", 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010979 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.416784+0000", 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011525933 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.422336+0000", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055522949999999996 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433810+0000", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011473524000000001 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433862+0000", 2024-09-14T04:08:41.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2414000000000003e-05 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433960+0000", 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7918000000000007e-05 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:180040 9.d 9:b3542a9b:::benchmark_data_smithi001_156540_object48446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.69763388299999, 2024-09-14T04:08:41.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.138239108, 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180040 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813429+0000", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813436+0000", 2024-09-14T04:08:41.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0389999999999999e-06 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813449+0000", 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3185e-05 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813457+0000", 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4100000000000002e-06 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813503+0000", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6489999999999997e-05 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813583+0000", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0097000000000002e-05 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813689+0000", 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105437 2024-09-14T04:08:41.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.817177+0000", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034882419999999999 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.817259+0000", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1843999999999998e-05 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951582+0000", 2024-09-14T04:08:41.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.134323266 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951632+0000", 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0160000000000001e-05 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951669+0000", 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7150000000000002e-05 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:180046 9.d 9:ba9a6653:::benchmark_data_smithi001_156540_object48452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.692929178, 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.136132161, 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180046 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.632 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818134+0000", 2024-09-14T04:08:41.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818139+0000", 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9409999999999998e-06 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818147+0000", 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0180000000000001e-06 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818151+0000", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5700000000000003e-06 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818184+0000", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2175999999999998e-05 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818282+0000", 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8043999999999998e-05 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818348+0000", 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6302000000000003e-05 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.821041+0000", 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026934300000000001 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.821810+0000", 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076834799999999997 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954149+0000", 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13233890200000001 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954232+0000", 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3170999999999994e-05 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954267+0000", 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5178000000000001e-05 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:220052 9.3 9:c193233b:::benchmark_data_smithi001_156540_object88458:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 71.508974609999996, 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.131223174, 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:08:41.638 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 220052 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:08:41.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002088+0000", 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002094+0000", 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5979999999999996e-06 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002105+0000", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0869e-05 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002111+0000", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3219999999999998e-06 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005294+0000", 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003182706 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005457+0000", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000163201 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005572+0000", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115293 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.009988+0000", 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044159469999999999 2024-09-14T04:08:41.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010128+0000", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014000300000000001 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133227+0000", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12309882 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133275+0000", 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7772000000000003e-05 2024-09-14T04:08:41.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133313+0000", 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7889999999999998e-05 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:08:41.644 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:08:41.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:41.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:41.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:41.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:41.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:41.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:41.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:42.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:42.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:42.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:42.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:42.178 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:42 smithi001 ceph-mon[107399]: pgmap v637: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:42.178 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:42 smithi001 ceph-mon[112165]: pgmap v637: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:42.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:42.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:42.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:08:42.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:42.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:42.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:42.536 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:42 smithi012 ceph-mon[101180]: pgmap v637: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:42.609 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:42.638 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:42.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:42.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:42.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:42.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:42.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:08:42.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:42.914 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:42.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:43.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:43.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:43.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:43.160 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_5' created 2024-09-14T04:08:43.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:43.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:43.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:43.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:43.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:08:43.391 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:43 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1854874338' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.391 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:43 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.392 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:43 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1854874338' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.392 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:43 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:43.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:43.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:43.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:43.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:43 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1854874338' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:43 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T04:08:43.649 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:43.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:43.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:43.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:43.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:43.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:43.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:44.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:44.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:44.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:44.165 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[107399]: pgmap v638: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:44.165 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T04:08:44.165 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[107399]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T04:08:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.185 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:44.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:44.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:44.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:44.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:44.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:44.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:44.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[112165]: pgmap v638: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T04:08:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:44 smithi001 ceph-mon[112165]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T04:08:44.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:44.565 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:44 smithi012 ceph-mon[101180]: pgmap v638: 83 pgs: 83 active+clean; 634 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:44 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T04:08:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:44 smithi012 ceph-mon[101180]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T04:08:44.667 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:44.689 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:44.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:44.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:44.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-14T04:08:44.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:44.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:44.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:45.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:45.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:45.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:45.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:45.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:45.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:45.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:45.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:45.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:45 smithi001 ceph-mon[107399]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T04:08:45.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:45 smithi001 ceph-mon[112165]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T04:08:45.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:45.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:45 smithi012 ceph-mon[101180]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T04:08:45.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:45.704 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:45.806 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:45.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:45.911 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:45.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:45.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:45.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:45.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:45.974 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:45.975 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:46.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:46.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:moved 0 / 297 (0%) 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:avg 37.125 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 4.19635 -> 4.19635 (expected baseline 5.69951) 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.2 with 31 -> 31 pgs (0.835017 -> 0.835017 * mean) 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.0 with 43 -> 43 pgs (1.15825 -> 1.15825 * mean) 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 0.05 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0066 2024-09-14T04:08:46.082 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0663 2024-09-14T04:08:46.083 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:08:46.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:46.207 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:46.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:46.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:46.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:46.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[107399]: pgmap v641: 99 pgs: 2 creating+peering, 13 unknown, 84 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:46.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:08:46.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[107399]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T04:08:46.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[112165]: pgmap v641: 99 pgs: 2 creating+peering, 13 unknown, 84 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:46.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:08:46.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-mon[112165]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T04:08:46.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.465 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:46.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:46.566 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:46.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:46 smithi012 ceph-mon[101180]: pgmap v641: 99 pgs: 2 creating+peering, 13 unknown, 84 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:46 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:08:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:46 smithi012 ceph-mon[101180]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T04:08:46.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:46.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:46.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:46.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:46.877 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:08:46.877 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:08:46.877 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:08:46.877 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:08:46.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:46.966 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:46.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:46] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.51.0" 2024-09-14T04:08:47.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:47.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:47.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:47.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:47.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:47.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:47.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:47.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:47 smithi001 ceph-mon[107399]: from='client.45008 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:47.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:47 smithi001 ceph-mon[112165]: from='client.45008 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:47.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:47.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:47.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:47 smithi012 ceph-mon[101180]: from='client.45008 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:47.600 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:47.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:08:47.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:47.726 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:47.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:47.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:47.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:47.977 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:47.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:48.079 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:48.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:48.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:48.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:48.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:48.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:48 smithi001 ceph-mon[107399]: pgmap v643: 99 pgs: 2 creating+peering, 2 unknown, 95 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:48 smithi001 ceph-mon[112165]: pgmap v643: 99 pgs: 2 creating+peering, 2 unknown, 95 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:48.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:48.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:48 smithi012 ceph-mon[101180]: pgmap v643: 99 pgs: 2 creating+peering, 2 unknown, 95 active+clean; 634 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:08:48.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:48.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:48.719 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:08:48.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 3.0 2024-09-14T04:08:48.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:48.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:48.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:48.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:48.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:48.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:49.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:49.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:49.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:49.230 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:49.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:49.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:49.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:49.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:49.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:49.615 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:49.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:49.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:49.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:49.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:49.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:49.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:49.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:50.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:50.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:50.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:50.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:50.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:50.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:50.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:50.460 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[107399]: pgmap v644: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:50.461 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:50.461 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:50.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[112165]: pgmap v644: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:50.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:50.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:50.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:50.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:50 smithi012 ceph-mon[101180]: pgmap v644: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:08:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:50.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:50.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:50.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:50.755 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:50.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:50.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:50.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:50.986 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:51.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:51.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:51.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:51.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:51.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:51.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:51.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:51.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:51.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:51.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:51.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:51.582 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:51.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:51.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:51.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:51.772 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:51.830 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:08:51.831 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:08:51.831 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:08:51.831 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:08:51.839 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:08:51.839 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:08:51.857 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:08:51.857 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:08:51.874 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:08:51.874 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:08:51.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:51.890 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:08:51.891 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:08:51.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:51.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:51.907 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:08:51.907 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:08:51.924 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:08:51.924 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:08:52.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:52.028 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:52.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:52.154 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:52.187 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[107399]: pgmap v645: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1401782445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[112165]: pgmap v645: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1401782445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:52.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:52.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:52.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:52.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:52.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:52.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:52 smithi012 ceph-mon[101180]: pgmap v645: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:08:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1401782445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:08:52.631 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:52.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:52.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:52.779 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:52.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:52.908 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:52.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:53.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:53.035 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:53.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:53.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:53.267 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:53.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:53.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:53.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:53.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:53.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[107399]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T04:08:53.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:53.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:53.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:53 smithi001 ceph-mon[112165]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T04:08:53.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:53.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:53.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:53 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:08:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2773204296' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:08:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:53 smithi012 ceph-mon[101180]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T04:08:53.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:53.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:53.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:53.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:53.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:53.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:53.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:53.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:54.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:54.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:54.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:54.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:54.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:54.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:54.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:54.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:54.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:54.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:54.399 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:54.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:54.942 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:54 smithi012 ceph-mon[101180]: pgmap v647: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:54.943 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:08:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:54.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:54.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:54 smithi001 ceph-mon[107399]: pgmap v647: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:54.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:08:54.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:54 smithi001 ceph-mon[112165]: pgmap v647: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:54.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:08:54.945 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-14T04:08:55.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:55.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:55.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:55.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:55.296 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:55.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:55.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:55.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:55.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:08:55.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:55 smithi001 ceph-mon[107399]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T04:08:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:08:55.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:55 smithi001 ceph-mon[112165]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T04:08:55.550 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:55.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3358697466' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:08:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:55 smithi012 ceph-mon[101180]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T04:08:55.676 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:55.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:55.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:55.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:55.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:55.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:55.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:56.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:56.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:56.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:56.206 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:56.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:56.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:56.339 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:56.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:08:56.405 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:56 smithi001 ceph-mon[107399]: pgmap v649: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:56.405 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:56 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:08:56.405 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:56 smithi001 ceph-mon[112165]: pgmap v649: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:56.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:56 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:08:56.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:56.465 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:56.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:56.560 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-14T04:08:56.563 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:08:56.563 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:08:56.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:56.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:56 smithi012 ceph-mon[101180]: pgmap v649: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:56 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:08:56.615 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:08:56.615 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:08:56.615 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_190881 2024-09-14T04:08:56.616 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:08:56.616 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:08:56.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.741 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:56.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:56.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:56.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:56.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:56.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:08:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:08:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:08:56] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:08:56.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:57.014 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:57.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:57.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:57.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:57.301 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:57.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:57.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:57.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:08:57.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:57.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:08:57.616 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 15 3759 3744 233.99 234 0.00554463 0.0042605 2024-09-14T04:08:57.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:57.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:57.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:57.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:08:57.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:57.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:08:57.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:57.966 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:08:57.966 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:08:58.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:58.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:58.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:58.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:08:58.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:08:58.385 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:58.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:58.469 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:58 smithi001 ceph-mon[107399]: pgmap v650: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:58.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:58 smithi001 ceph-mon[112165]: pgmap v650: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:58.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:58.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:58 smithi012 ceph-mon[101180]: pgmap v650: 99 pgs: 99 active+clean; 634 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:08:58.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 7498 7482 233.756 233.625 0.00356411 0.00427089 2024-09-14T04:08:58.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:58.705 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:58.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:58.823 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:58.828 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:08:58.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:58.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:58.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:58.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:59.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:08:59.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:08:59.216 INFO:teuthology.orchestra.run.smithi001.stdout:moved 35 / 297 (11.7845%) 2024-09-14T04:08:59.216 INFO:teuthology.orchestra.run.smithi001.stdout:avg 37.125 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 4.19635 -> 9.35999 (expected baseline 5.69951) 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.2 with 31 -> 38 pgs (0.835017 -> 1.02357 * mean) 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.0 with 43 -> 20 pgs (1.15825 -> 0.538721 * mean) 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0067 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0074 2024-09-14T04:08:59.217 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 weight 1.0000 -> 0.4363 2024-09-14T04:08:59.218 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 weight 1.0000 -> 0.7367 2024-09-14T04:08:59.218 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 weight 1.0000 -> 0.8981 2024-09-14T04:08:59.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:08:59.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.259 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:59.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:08:59.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:59.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:08:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"} : dispatch 2024-09-14T04:08:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:08:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"} : dispatch 2024-09-14T04:08:59.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:08:59.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:08:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:08:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"} : dispatch 2024-09-14T04:08:59.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:08:59.616 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 9466 9450 196.837 123 0.00134747 0.00443421 2024-09-14T04:08:59.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:08:59.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:08:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:08:59.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:59.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:59.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:08:59.886 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:08:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:59.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:08:59.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:00.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:00.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:00.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:00.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:00.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:00.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:00.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:00.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:00.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:00.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:00.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[107399]: pgmap v651: 99 pgs: 99 active+clean; 68 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s wr, 134 op/s 2024-09-14T04:09:00.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[107399]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"}]': finished 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[107399]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[112165]: pgmap v651: 99 pgs: 99 active+clean; 68 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s wr, 134 op/s 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[112165]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"}]': finished 2024-09-14T04:09:00.540 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:00 smithi001 ceph-mon[112165]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T04:09:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:00 smithi012 ceph-mon[101180]: pgmap v651: 99 pgs: 99 active+clean; 68 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s wr, 134 op/s 2024-09-14T04:09:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:00 smithi012 ceph-mon[101180]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:00 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'28596','5':'48281','7':'58858'}"}]': finished 2024-09-14T04:09:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:00 smithi012 ceph-mon[101180]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T04:09:00.616 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:00.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 9466 9450 147.629 0 - 0.00443421 2024-09-14T04:09:00.640 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:00.742 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:00.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:00.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:00.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:00.994 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:01.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:01.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:01.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:01.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:01.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:01.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:01.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:01.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:01.531 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:01.570 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:01.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:01 smithi012 ceph-mon[101180]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T04:09:01.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 10522 10506 131.304 33 0.00610865 0.00760883 2024-09-14T04:09:01.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:01.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:01 smithi001 ceph-mon[112165]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T04:09:01.698 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:01 smithi001 ceph-mon[107399]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T04:09:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:01.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:01.745 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:09:01.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-14T04:09:01.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:01.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:01.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:01.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:02.004 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:02.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:02.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:02.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:02.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:02.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:02.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:02.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:02.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:02.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:02 smithi001 ceph-mon[112165]: pgmap v654: 99 pgs: 6 remapped+peering, 5 peering, 88 active+clean; 448 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 13 B/s, 1 objects/s recovering 2024-09-14T04:09:02.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:02 smithi001 ceph-mon[112165]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T04:09:02.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:02 smithi001 ceph-mon[107399]: pgmap v654: 99 pgs: 6 remapped+peering, 5 peering, 88 active+clean; 448 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 13 B/s, 1 objects/s recovering 2024-09-14T04:09:02.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:02 smithi001 ceph-mon[107399]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T04:09:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:02 smithi012 ceph-mon[101180]: pgmap v654: 99 pgs: 6 remapped+peering, 5 peering, 88 active+clean; 448 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 13 B/s, 1 objects/s recovering 2024-09-14T04:09:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:02 smithi012 ceph-mon[101180]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T04:09:02.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 13229 13213 137.609 169.188 0.00524448 0.00725896 2024-09-14T04:09:02.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:02.700 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:02.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:02.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:02.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:02.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:02.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:02.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:03.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:03.112 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:03.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:03.241 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:03.277 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:03.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:03.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:03 smithi001 ceph-mon[107399]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T04:09:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:03 smithi001 ceph-mon[107399]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T04:09:03.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:03 smithi001 ceph-mon[112165]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T04:09:03.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:03 smithi001 ceph-mon[112165]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T04:09:03.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:03.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:03 smithi012 ceph-mon[101180]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T04:09:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:03 smithi012 ceph-mon[101180]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T04:09:03.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 15783 15767 140.752 159.625 0.00489092 0.00709831 2024-09-14T04:09:03.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:03.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:03.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.849 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:03.950 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:03.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:03.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:04.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:04.125 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:04.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:04.269 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:04.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:04.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:04.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:04 smithi001 ceph-mon[112165]: pgmap v657: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 5 peering, 84 active+clean; 611 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 19 KiB/s rd, 136 MiB/s wr, 2.18k op/s; 449/29370 objects misplaced (1.529%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:09:04.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:04 smithi001 ceph-mon[112165]: 1.2 scrub starts 2024-09-14T04:09:04.503 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:04 smithi001 ceph-mon[107399]: pgmap v657: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 5 peering, 84 active+clean; 611 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 19 KiB/s rd, 136 MiB/s wr, 2.18k op/s; 449/29370 objects misplaced (1.529%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:09:04.503 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:04 smithi001 ceph-mon[107399]: 1.2 scrub starts 2024-09-14T04:09:04.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:04.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:04 smithi012 ceph-mon[101180]: pgmap v657: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 5 peering, 84 active+clean; 611 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 19 KiB/s rd, 136 MiB/s wr, 2.18k op/s; 449/29370 objects misplaced (1.529%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:09:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:04 smithi012 ceph-mon[101180]: 1.2 scrub starts 2024-09-14T04:09:04.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 18328 18312 143.039 159.062 0.00466306 0.00698402 2024-09-14T04:09:04.650 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:04.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:04.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:04.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:04.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:04.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:04.980 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:04.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:05.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:05.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:05.193 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:09:05.193 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:09:05.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:05.233 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:09:05.233 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:09:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:05.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:05.278 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:09:05.278 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3100473965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4051621951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3100473965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.279 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.280 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4051621951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:05.313 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:09:05.314 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:09:05.353 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:09:05.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:09:05.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:05.393 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:09:05.394 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:09:05.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:05.429 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:09:05.429 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:09:05.462 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:09:05.462 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:09:05.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:05.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:09:05.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:05.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:05.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3100473965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:09:05.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4051621951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:05.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 20674 20658 143.436 146.625 0.00921046 0.00696585 2024-09-14T04:09:05.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:05.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:05.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:05.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:05.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:05.937 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:09:05.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:05.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:06.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:06.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:06.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:06.217 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:06.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:06.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:06 smithi012 ceph-mon[101180]: pgmap v659: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 89 active+clean; 673 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 31 KiB/s rd, 117 MiB/s wr, 1.89k op/s; 449/32355 objects misplaced (1.388%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:09:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:06 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:06 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:09:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:06 smithi012 ceph-mon[101180]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T04:09:06.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:06.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:06.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:06.600 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:06.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 21685 21669 135.411 63.1875 0.00165694 0.00697296 2024-09-14T04:09:06.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[107399]: pgmap v659: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 89 active+clean; 673 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 31 KiB/s rd, 117 MiB/s wr, 1.89k op/s; 449/32355 objects misplaced (1.388%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[107399]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[112165]: pgmap v659: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+undersized+remapped, 6 remapped+peering, 89 active+clean; 673 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 31 KiB/s rd, 117 MiB/s wr, 1.89k op/s; 449/32355 objects misplaced (1.388%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:09:06.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:09:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-mon[112165]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T04:09:06.727 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:06.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:06.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:06.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:06.968 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:06] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.51.0" 2024-09-14T04:09:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:06.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:06.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:07.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:07.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:07.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:07.245 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.347 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262008 9.b 9.b4892acb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.330280+0000", 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.857891502999998, 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98178777399999995, 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262008 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.348 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330280+0000", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330280+0000", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330282+0000", 2024-09-14T04:09:07.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0849999999999999e-06 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330288+0000", 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7860000000000002e-06 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330300+0000", 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1639e-05 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330308+0000", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0050000000000004e-06 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.348396+0000", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018087855 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.348406+0000", 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0025e-05 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.312041+0000", 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96363487599999997 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.312068+0000", 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7503000000000001e-05 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262009 9.8 9.38909dd8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.330320+0000", 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.857851898, 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97495783999999996, 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262009 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.353 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330320+0000", 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330320+0000", 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330320+0000", 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.89e-07 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330323+0000", 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1710000000000001e-06 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330329+0000", 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8580000000000002e-06 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330338+0000", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1419999999999993e-06 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330390+0000", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2669e-05 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.330401+0000", 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305263+0000", 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97486188100000004 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305278+0000", 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5209e-05 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.357 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262019 9.5 9.f09d0285 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.339732+0000", 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.848439751000001, 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.969597657, 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262019 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.358 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339732+0000", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339732+0000", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339734+0000", 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.689e-06 2024-09-14T04:09:07.359 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339740+0000", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1739999999999997e-06 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339753+0000", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3512999999999999e-05 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339762+0000", 2024-09-14T04:09:07.360 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2360000000000004e-06 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339917+0000", 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015493700000000001 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.339927+0000", 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0485999999999999e-05 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.361 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309312+0000", 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.969385053 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309330+0000", 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7569000000000001e-05 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262022 9.9 9.a8a96599 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.347855+0000", 2024-09-14T04:09:07.362 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.840316412, 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95939149099999999, 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262022 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347855+0000", 2024-09-14T04:09:07.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347855+0000", 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347858+0000", 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2010000000000002e-06 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347863+0000", 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6130000000000003e-06 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347877+0000", 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3341e-05 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347887+0000", 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0066e-05 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347949+0000", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2112000000000004e-05 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.347961+0000", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.238e-05 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.307235+0000", 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95927425799999999 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.307247+0000", 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.152e-05 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262024 9.8 9.33774118 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.350520+0000", 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.837652149, 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95479605300000003, 2024-09-14T04:09:07.367 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262024 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350520+0000", 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.368 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350520+0000", 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350521+0000", 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.668e-06 2024-09-14T04:09:07.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350528+0000", 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3010000000000003e-06 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350546+0000", 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8743999999999999e-05 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350556+0000", 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6390000000000004e-06 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350900+0000", 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034368300000000001 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.350910+0000", 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8900000000000002e-06 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305306+0000", 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95439684800000002 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305316+0000", 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2799999999999992e-06 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262028 9.9 9.a14db0f9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.372 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.353368+0000", 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.834803581000003, 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96058239099999998, 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262028 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353368+0000", 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353368+0000", 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353370+0000", 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9230000000000001e-06 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353378+0000", 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1880000000000002e-06 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353398+0000", 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0132999999999998e-05 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353408+0000", 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8600000000000005e-06 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353475+0000", 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6951000000000003e-05 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.353486+0000", 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1005999999999999e-05 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.313937+0000", 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96045039099999996 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.313951+0000", 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3939e-05 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.377 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262031 9.8 9.6fa22bc8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.357132+0000", 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.831040205000001, 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.948222185, 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262031 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357132+0000", 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357132+0000", 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357134+0000", 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0439999999999998e-06 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357141+0000", 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1799999999999999e-06 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357158+0000", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7598e-05 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357169+0000", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0902e-05 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357247+0000", 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8176999999999994e-05 2024-09-14T04:09:07.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.357277+0000", 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9657000000000001e-05 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305343+0000", 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94806601700000004 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305354+0000", 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.061e-05 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262036 9.8 9.853472b8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.362421+0000", 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.825750648000003, 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94297178800000003, 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.383 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262036 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362421+0000", 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362421+0000", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362423+0000", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1129999999999999e-06 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362433+0000", 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8800000000000003e-06 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362451+0000", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8040999999999999e-05 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362459+0000", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9850000000000006e-06 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362533+0000", 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3471000000000002e-05 2024-09-14T04:09:07.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.362542+0000", 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2240000000000001e-06 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305381+0000", 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94283924200000002 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.305393+0000", 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1832e-05 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262038 9.b 9.d058bcbb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.366681+0000", 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.821490550999997, 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95225177299999997, 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.388 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262038 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366681+0000", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366681+0000", 2024-09-14T04:09:07.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366683+0000", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0310000000000001e-06 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366691+0000", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0649999999999999e-06 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366710+0000", 2024-09-14T04:09:07.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8692999999999999e-05 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366719+0000", 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4210000000000001e-06 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366791+0000", 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1701999999999999e-05 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.366804+0000", 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2539e-05 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.318920+0000", 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95211612499999998 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.318933+0000", 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3196999999999999e-05 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.392 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262041 9.b 9.be5a604b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.367279+0000", 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.820893155999997, 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95169305500000001, 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262041 2024-09-14T04:09:07.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367279+0000", 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367279+0000", 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367280+0000", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1999999999999998e-07 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367284+0000", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4390000000000002e-06 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367297+0000", 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3138e-05 2024-09-14T04:09:07.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367304+0000", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.951e-06 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367369+0000", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5075999999999998e-05 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367378+0000", 2024-09-14T04:09:07.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3190000000000001e-06 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.318962+0000", 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95158317000000003 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.318972+0000", 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0042e-05 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262042 9.5 9.2c6d2b55 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.367775+0000", 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.820396813999999, 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.941891429, 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262042 2024-09-14T04:09:07.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367775+0000", 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367775+0000", 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367776+0000", 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0580000000000001e-06 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367781+0000", 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9320000000000004e-06 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367796+0000", 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4732000000000001e-05 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367804+0000", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8790000000000003e-06 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367879+0000", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5044999999999997e-05 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.367888+0000", 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3209999999999994e-06 2024-09-14T04:09:07.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309656+0000", 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94176789599999999 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309666+0000", 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0566e-05 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.402 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262048 9.5 9.c57062d5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.375255+0000", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.812916411000003, 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93444729599999998, 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262048 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375255+0000", 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375255+0000", 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375257+0000", 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.03e-06 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375264+0000", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3679999999999998e-06 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375283+0000", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9145999999999999e-05 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375293+0000", 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9329999999999996e-06 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375374+0000", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0816999999999996e-05 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375381+0000", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3780000000000003e-06 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309694+0000", 2024-09-14T04:09:07.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93431260400000005 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309703+0000", 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.02e-06 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262049 9.b 9.6f440aab (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.375312+0000", 2024-09-14T04:09:07.407 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.812860143000002, 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94369586100000002, 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262049 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375312+0000", 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375312+0000", 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375312+0000", 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.22e-07 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375314+0000", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5380000000000001e-06 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375321+0000", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1859999999999997e-06 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375325+0000", 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0489999999999997e-06 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375422+0000", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7213000000000006e-05 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.375435+0000", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3081999999999999e-05 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.318997+0000", 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94356196800000003 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.319007+0000", 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0502999999999999e-05 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262053 9.b 9.2102f46b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.382251+0000", 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.805920462000003, 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93679316499999998, 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.412 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262053 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382251+0000", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382251+0000", 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382253+0000", 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.905e-06 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382262+0000", 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4400000000000005e-06 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382279+0000", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7073999999999999e-05 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382286+0000", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1559999999999998e-06 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382340+0000", 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3798000000000002e-05 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382347+0000", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7870000000000002e-06 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.319034+0000", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93668654699999998 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.319044+0000", 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0458e-05 2024-09-14T04:09:07.416 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262054 9.5 9.3841cdc5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.382480+0000", 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.805691803999999, 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.92725676599999995, 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262054 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382480+0000", 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382480+0000", 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.418 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382480+0000", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1600000000000001e-07 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382483+0000", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4880000000000001e-06 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382490+0000", 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4760000000000001e-06 2024-09-14T04:09:07.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382494+0000", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9450000000000003e-06 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382536+0000", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1264000000000003e-05 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.382541+0000", 2024-09-14T04:09:07.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2109999999999997e-06 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309728+0000", 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.92718695699999998 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309737+0000", 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9090000000000006e-06 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:262058 9.5 9.db896a85 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:14.395778+0000", 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.792393502000003, 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.91399870800000005, 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.422 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 262058 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395778+0000", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395778+0000", 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395780+0000", 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9400000000000001e-06 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395787+0000", 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4370000000000004e-06 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395803+0000", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6748999999999998e-05 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395811+0000", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2869999999999999e-06 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395889+0000", 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8753999999999998e-05 2024-09-14T04:09:07.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:14.395898+0000", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2400000000000007e-06 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309767+0000", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.91386936299999999 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:15.309777+0000", 2024-09-14T04:09:07.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9380000000000004e-06 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:262062 9.b e175/174 9:db855ab8:::benchmark_data_smithi001_156540_object130468:head v 175'16302, mlcod=0'0)", 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:16.654966+0000", 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 50.533206114000002, 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10256979200000001, 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.654966+0000", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.654966+0000", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.654967+0000", 2024-09-14T04:09:07.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7549999999999999e-06 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.654979+0000", 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1793999999999999e-05 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.654996+0000", 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6728e-05 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.655005+0000", 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4720000000000001e-06 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.685603+0000", 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030597137999999999 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.685635+0000", 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2638000000000002e-05 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.757064+0000", 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.071428333999999996 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:16.757535+0000", 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00047193300000000001 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:262924 9.5 e175/174 9:a7ddf137:::benchmark_data_smithi001_156540_object131330:head v 175'16452, mlcod=151'10151)", 2024-09-14T04:09:07.431 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:17.690876+0000", 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 49.497295825999998, 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13401086600000001, 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690876+0000", 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.432 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690876+0000", 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690877+0000", 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3400000000000001e-06 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690886+0000", 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8929999999999994e-06 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690906+0000", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9709999999999999e-05 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.690915+0000", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9400000000000008e-06 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.736541+0000", 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.045626443000000003 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.736571+0000", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9592999999999999e-05 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.824820+0000", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.088248654999999995 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.824887+0000", 2024-09-14T04:09:07.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7292000000000006e-05 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:262933 9.5 e175/174 9:ad3356a2:::benchmark_data_smithi001_156540_object131339:head v 175'16453, mlcod=151'10151)", 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:17.703529+0000", 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 49.484642657000002, 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.121444283, 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703529+0000", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703529+0000", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703530+0000", 2024-09-14T04:09:07.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1200000000000002e-07 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703534+0000", 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5560000000000001e-06 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703545+0000", 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0695e-05 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.703551+0000", 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6400000000000002e-06 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.749441+0000", 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.045890463999999999 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.749465+0000", 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4287e-05 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.824892+0000", 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075426901000000005 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.824973+0000", 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1027999999999997e-05 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.440 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.44714.0:262960 9.5 e175/174 9:a27e5265:::benchmark_data_smithi001_156540_object131366:head v 175'16454, mlcod=151'10151)", 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:08:17.738978+0000", 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 49.449193944000001, 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.11203372, 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.738978+0000", 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.738978+0000", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.738978+0000", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1799999999999995e-07 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.738983+0000", 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.549e-06 2024-09-14T04:09:07.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.738995+0000", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1667e-05 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.739000+0000", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.575e-06 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.762089+0000", 2024-09-14T04:09:07.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.023088989000000001 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.762106+0000", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6566999999999999e-05 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.850952+0000", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.088846066000000001 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:08:17.851012+0000", 2024-09-14T04:09:07.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9688999999999997e-05 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:07.445 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:09:07.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:07.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:07.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:07.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:07.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:07.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 21864 21848 124.119 11.1875 0.00187586 0.00759674 2024-09-14T04:09:07.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:07.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:07.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:07.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:07.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:07.980 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:07.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:07.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:08.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:08.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:08.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:08.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:07 smithi001 ceph-mon[112165]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T04:09:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:07 smithi001 ceph-mon[112165]: pgmap v662: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 114 MiB/s wr, 1.84k op/s; 1748/56139 objects degraded (3.114%); 548/56139 objects misplaced (0.976%); 6.3 MiB/s, 100 objects/s recovering 2024-09-14T04:09:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:07 smithi001 ceph-mon[107399]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T04:09:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:07 smithi001 ceph-mon[107399]: pgmap v662: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 114 MiB/s wr, 1.84k op/s; 1748/56139 objects degraded (3.114%); 548/56139 objects misplaced (0.976%); 6.3 MiB/s, 100 objects/s recovering 2024-09-14T04:09:08.230 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:08.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:07 smithi012 ceph-mon[101180]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T04:09:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:07 smithi012 ceph-mon[101180]: pgmap v662: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 114 MiB/s wr, 1.84k op/s; 1748/56139 objects degraded (3.114%); 548/56139 objects misplaced (0.976%); 6.3 MiB/s, 100 objects/s recovering 2024-09-14T04:09:08.354 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:08.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:08.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:08.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:08.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:08.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:08.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 22005 21989 114.511 8.8125 0.00191901 0.00826409 2024-09-14T04:09:08.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:08.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:08.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:08.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:08.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:08 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 1748/56139 objects degraded (3.114%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:08.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:08 smithi001 ceph-mon[107399]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T04:09:08.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:08.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:08.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:09.096 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:09.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:08 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 1748/56139 objects degraded (3.114%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:08 smithi001 ceph-mon[112165]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T04:09:09.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:09.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:08 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 1748/56139 objects degraded (3.114%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:08 smithi012 ceph-mon[101180]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T04:09:09.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:09.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:09.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:09.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:09.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:09.617 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 22397 22381 107.587 24.5 0.00182643 0.0088931 2024-09-14T04:09:09.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:09.726 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:09.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:09.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:09.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:09.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:09.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:10.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:10.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:10.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:10.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:09 smithi001 ceph-mon[107399]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T04:09:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:09 smithi001 ceph-mon[107399]: pgmap v665: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s; 1748/62961 objects degraded (2.776%); 99/62961 objects misplaced (0.157%); 15 MiB/s, 233 objects/s recovering 2024-09-14T04:09:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:09 smithi001 ceph-mon[112165]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T04:09:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:09 smithi001 ceph-mon[112165]: pgmap v665: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s; 1748/62961 objects degraded (2.776%); 99/62961 objects misplaced (0.157%); 15 MiB/s, 233 objects/s recovering 2024-09-14T04:09:10.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:10.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:09 smithi012 ceph-mon[101180]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T04:09:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:09 smithi012 ceph-mon[101180]: pgmap v665: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 159 MiB/s wr, 2.55k op/s; 1748/62961 objects degraded (2.776%); 99/62961 objects misplaced (0.157%); 15 MiB/s, 233 objects/s recovering 2024-09-14T04:09:10.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:10.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:10.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.531 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:10.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:10.618 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 23278 23262 103.835 55.0625 0.00645001 0.00962485 2024-09-14T04:09:10.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:10.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:10.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:10.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:10.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:10.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:10.952 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:10.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:11.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:11.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:11.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[107399]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[112165]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:09:11.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:10 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:09:11.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:11.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:10 smithi012 ceph-mon[101180]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T04:09:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:10 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:09:11.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:10 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:09:11.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:11.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:11.516 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:11.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:11.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 25402 25386 105.761 132.75 0.00499537 0.00944913 2024-09-14T04:09:11.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:11.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:11.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:11.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:11.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:11.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:12.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:12.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:12.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:12.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[107399]: pgmap v667: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 34 MiB/s, 549 objects/s recovering 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[112165]: pgmap v667: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 34 MiB/s, 549 objects/s recovering 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:09:12.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:12.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:12.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:11 smithi012 ceph-mon[101180]: pgmap v667: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 34 MiB/s, 549 objects/s recovering 2024-09-14T04:09:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:09:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:12.366 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:12.469 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:12.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:12.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:12.618 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 27681 27665 108.052 142.438 0.00870043 0.00924871 2024-09-14T04:09:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:12.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:12.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:12.897 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:12.930 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:12.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:13.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:13.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:13.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:13.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:12 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1748/62961 objects degraded (2.776%), 3 pgs degraded) 2024-09-14T04:09:13.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:12 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1748/62961 objects degraded (2.776%), 3 pgs degraded) 2024-09-14T04:09:13.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:13.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:13.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:12 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1748/62961 objects degraded (2.776%), 3 pgs degraded) 2024-09-14T04:09:13.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:13.468 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:13.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:13.616 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:13.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 30033 30017 110.342 147 0.00879326 0.00905719 2024-09-14T04:09:13.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:13.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:13.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:13.898 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:13.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:13.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:13.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:14.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.082 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:14.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:14.153 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:13 smithi001 ceph-mon[107399]: pgmap v668: 99 pgs: 99 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 24 MiB/s, 382 objects/s recovering 2024-09-14T04:09:14.153 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:13 smithi001 ceph-mon[112165]: pgmap v668: 99 pgs: 99 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 24 MiB/s, 382 objects/s recovering 2024-09-14T04:09:14.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:14.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:14.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:14.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:13 smithi012 ceph-mon[101180]: pgmap v668: 99 pgs: 99 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 24 MiB/s, 382 objects/s recovering 2024-09-14T04:09:14.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.366 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:14.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:14.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:14.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:14.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:14.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 32282 32266 112.016 140.562 0.00957221 0.00892065 2024-09-14T04:09:14.643 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:14.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:14.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:09:14.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:14.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:14.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:14.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:14.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:15.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:15.074 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:15.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:15.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:15.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:15.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:15.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:15.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:15.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:15.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:09:15.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:15.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 34323 34307 112.834 127.562 0.00627644 0.00885766 2024-09-14T04:09:15.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:15.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:09:15.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:15.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:15.845 INFO:teuthology.orchestra.run.smithi001.stdout:moved 20 / 297 (6.73401%) 2024-09-14T04:09:15.845 INFO:teuthology.orchestra.run.smithi001.stdout:avg 37.125 2024-09-14T04:09:15.845 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 9.35999 -> 12.17 (expected baseline 5.69951) 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.0 with 20 -> 10 pgs (0.538721 -> 0.26936 * mean) 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.1 with 45 -> 48 pgs (1.21212 -> 1.29293 * mean) 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:09:15.846 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0130 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0143 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 weight 0.4363 -> 0.2532 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 weight 1.0000 -> 0.8248 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 weight 0.8981 -> 1.0000 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 weight 0.7367 -> 0.7816 2024-09-14T04:09:15.847 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:09:15.884 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:15.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:15.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:15.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:15 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:09:15.842+0000 7fc9b3f23640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-09-14T04:09:16.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:16.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:16.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:16.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:16.309 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:16.328 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:16.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:16 smithi001 ceph-mon[112165]: pgmap v669: 99 pgs: 99 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 39 MiB/s wr, 622 op/s; 16 MiB/s, 257 objects/s recovering 2024-09-14T04:09:16.348 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:16 smithi001 ceph-mon[107399]: pgmap v669: 99 pgs: 99 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 39 MiB/s wr, 622 op/s; 16 MiB/s, 257 objects/s recovering 2024-09-14T04:09:16.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:16.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:16.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:16 smithi012 ceph-mon[101180]: pgmap v669: 99 pgs: 99 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 39 MiB/s wr, 622 op/s; 16 MiB/s, 257 objects/s recovering 2024-09-14T04:09:16.597 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:16.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:09:16.618962+0000 min lat: 0.00110309 max lat: 2.05165 avg lat: 0.0088417 2024-09-14T04:09:16.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:09:16.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 36191 36175 113.029 116.75 0.0067414 0.0088417 2024-09-14T04:09:16.635 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:16.703 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:16] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.51.0" 2024-09-14T04:09:16.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:16.772 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:16.874 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:16.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:16.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:16.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:17.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:17.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:17.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:17.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:17.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:17.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:17.307 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:09:17.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-09-14T04:09:17.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:17.393 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:17.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:17.452 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:17 smithi012 ceph-mon[101180]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:17 smithi001 ceph-mon[107399]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:17 smithi001 ceph-mon[112165]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:17.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:17.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:17.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:17.619 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 38195 38179 113.61 125.25 0.00740896 0.00879748 2024-09-14T04:09:17.636 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:17.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:17.772 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:17.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:17.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:17.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:17.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:18.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:18.054 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:18.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:18.202 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:18.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:18.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:18.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:18.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[107399]: pgmap v670: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 14 MiB/s, 225 objects/s recovering 2024-09-14T04:09:18.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.391 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.391 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[112165]: pgmap v670: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 14 MiB/s, 225 objects/s recovering 2024-09-14T04:09:18.391 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.392 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:18.504 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:18 smithi012 ceph-mon[101180]: pgmap v670: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 14 MiB/s, 225 objects/s recovering 2024-09-14T04:09:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:18.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:18.620 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 40060 40044 113.74 116.562 0.0101037 0.00878632 2024-09-14T04:09:18.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:18.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:18.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:18.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:18.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:18.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:19.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.064 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:19.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:19.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:19.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:19.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:19.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:19.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:19.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:19.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.620 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:19.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:19.621 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 42133 42117 114.428 129.562 0.0056581 0.00873341 2024-09-14T04:09:19.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:19.755 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:19.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:19.894 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:19.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:19.983 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:19.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:20.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:20.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:20.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:20.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:20.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:20.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:20 smithi001 ceph-mon[112165]: pgmap v671: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 12 MiB/s, 184 objects/s recovering 2024-09-14T04:09:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:20 smithi001 ceph-mon[107399]: pgmap v671: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 12 MiB/s, 184 objects/s recovering 2024-09-14T04:09:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:20.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:20.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:20 smithi012 ceph-mon[101180]: pgmap v671: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 12 MiB/s, 184 objects/s recovering 2024-09-14T04:09:20.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:20.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:20.620 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 44097 44081 114.774 122.75 0.0118989 0.0087072 2024-09-14T04:09:20.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:20.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:20.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:20.859 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:20.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:20.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:20.998 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:21.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.033 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:21.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:21.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:21.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:21.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:21.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:21.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:21.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:21.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:21.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:21.620 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 45710 45694 114.215 100.812 0.00796716 0.00875009 2024-09-14T04:09:21.643 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:21.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.712 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:21.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:21.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:21.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:21.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:21.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:22.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:22.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:22.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:22.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:22 smithi001 ceph-mon[107399]: pgmap v672: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10 MiB/s, 165 objects/s recovering 2024-09-14T04:09:22.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:22 smithi001 ceph-mon[112165]: pgmap v672: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10 MiB/s, 165 objects/s recovering 2024-09-14T04:09:22.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:22.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:22.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:22.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:22.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:22.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:22 smithi012 ceph-mon[101180]: pgmap v672: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10 MiB/s, 165 objects/s recovering 2024-09-14T04:09:22.620 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 47519 47503 114.17 113.062 0.00841372 0.0087534 2024-09-14T04:09:22.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:22.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:22.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:22.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:22.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:22.958 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:09:22.958 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:09:22.959 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:09:22.960 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:09:22.960 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:09:22.960 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:09:22.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:23.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:23.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:23.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:23.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:23.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:23.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:23.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:23.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:23.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:23 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/711952187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:23.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:23 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/711952187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:23.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:23.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:23 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/711952187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:23.621 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 49355 49339 114.191 114.75 0.00412082 0.00875224 2024-09-14T04:09:23.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:23.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:23.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:23.840 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:23.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:23.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:23.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:23.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:24.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.117 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:24.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:24.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:24.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:24.357 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:24.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:24 smithi001 ceph-mon[107399]: pgmap v673: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:24 smithi001 ceph-mon[112165]: pgmap v673: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:24.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:24.534 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:24 smithi012 ceph-mon[101180]: pgmap v673: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:24.621 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 51000 50984 113.784 102.812 0.00771362 0.00878297 2024-09-14T04:09:24.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.674 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:24.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:24.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:24.811 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:24.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:24.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:24.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:24.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:25.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:25.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:25.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:25.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:25.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:25.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:25.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:25.513 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:25.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:25.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:25.620 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 52565 52549 113.233 97.8125 0.0101682 0.0088265 2024-09-14T04:09:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.653 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:25.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:25.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:25.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:25.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:25.931 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:25.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:26.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:26.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:26.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:26.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:26.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:26.343 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:09:26.343 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:09:26.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:26.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:26.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:26 smithi001 ceph-mon[107399]: pgmap v674: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:26 smithi001 ceph-mon[112165]: pgmap v674: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:26.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:26.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:26 smithi012 ceph-mon[101180]: pgmap v674: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-14T04:09:26.621 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 54386 54370 113.252 113.812 0.0107269 0.00882489 2024-09-14T04:09:26.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:26.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:26.777 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:26.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:26.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:26.954 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:26.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:26.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:26.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:26] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-09-14T04:09:27.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:27.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:27.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:27.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:27.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:27.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:27.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:27.487 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:27.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:27.621 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 56152 56136 113.159 110.375 0.00609468 0.00883228 2024-09-14T04:09:27.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:27.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:27.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:27.806 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:27.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:27.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:27.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:27.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:28.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:28.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:28.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:09:28.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:28.227 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:28.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:28.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:28.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:09:28.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:28.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:28 smithi001 ceph-mon[107399]: pgmap v675: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:09:28.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:28 smithi001 ceph-mon[112165]: pgmap v675: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:09:28.515 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:28 smithi012 ceph-mon[101180]: pgmap v675: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:09:28.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:28.623 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 57778 57762 112.792 101.625 0.00539012 0.00886001 2024-09-14T04:09:28.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:28.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:28.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:28.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:28.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:28.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:28.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:29.022 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:29.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:29.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:29.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:29.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:29.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:29.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:29.487 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:29.589 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:29.623 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 59384 59368 112.415 100.375 0.012791 0.00889009 2024-09-14T04:09:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:29.747 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:29.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:29.934 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:29.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:30.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:30.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:30.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:30.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:30.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:30.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:30 smithi001 ceph-mon[107399]: pgmap v676: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-14T04:09:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:30 smithi001 ceph-mon[112165]: pgmap v676: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-14T04:09:30.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:30.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:30.590 INFO:teuthology.orchestra.run.smithi001.stdout:moved 7 / 297 (2.3569%) 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:avg 37.125 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 9.35999 -> 9.54512 (expected baseline 5.69951) 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.0 with 20 -> 18 pgs (0.538721 -> 0.484848 * mean) 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.1 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 0.05 2024-09-14T04:09:30.591 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0202 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0223 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 weight 0.4363 -> 0.3864 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 weight 1.0000 -> 0.9500 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 weight 0.7367 -> 0.7867 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 weight 0.8981 -> 0.9481 2024-09-14T04:09:30.592 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:09:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:30 smithi012 ceph-mon[101180]: pgmap v676: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-14T04:09:30.623 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 60818 60802 111.744 89.625 0.00701898 0.00894344 2024-09-14T04:09:30.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:30.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:30.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:30.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:30.890 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:09:30.588+0000 7fc9b3f23640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-09-14T04:09:30.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:30.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:30.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:31.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.087 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:31.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:31.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:31.224 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:09:31.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:31.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:31.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:31.362 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:31.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:31.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1074808486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:31 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:31.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 62281 62265 111.162 91.4375 0.00591255 0.0089904 2024-09-14T04:09:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.641 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:31.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1074808486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:31 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1074808486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:31 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:09:31.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:31.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:31.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:31.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:31.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:32.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:32.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:32.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:32.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:32.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:32.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:32.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:32.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: from='client.54814 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: pgmap v677: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[107399]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T04:09:32.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: from='client.54814 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: pgmap v677: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:09:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:09:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:09:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:09:32.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:32 smithi001 ceph-mon[112165]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T04:09:32.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:32.591 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:32.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: from='client.54814 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: pgmap v677: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:09:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:32 smithi012 ceph-mon[101180]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T04:09:32.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 63528 63512 110.239 77.9375 0.0108857 0.00906657 2024-09-14T04:09:32.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:32.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:32.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:32.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:32.869 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:32.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:32.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:33.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:33.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:33.142 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:33.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:33.272 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:33.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:33.374 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:33.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:33.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:33.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:33.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 65034 65018 109.803 94.125 0.00820847 0.00910194 2024-09-14T04:09:33.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:33.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:33.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:33.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:33.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:33.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:33.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:34.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:34.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:34.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:34.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:34.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:34.402 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[107399]: pgmap v679: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[107399]: 1.2 scrub starts 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[107399]: 1.2 scrub ok 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[112165]: pgmap v679: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[112165]: 1.2 scrub starts 2024-09-14T04:09:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:34 smithi001 ceph-mon[112165]: 1.2 scrub ok 2024-09-14T04:09:34.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:34.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:34 smithi012 ceph-mon[101180]: pgmap v679: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-14T04:09:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:34 smithi012 ceph-mon[101180]: 1.2 scrub starts 2024-09-14T04:09:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:34 smithi012 ceph-mon[101180]: 1.2 scrub ok 2024-09-14T04:09:34.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 66599 66583 109.488 97.8125 0.0122086 0.00912848 2024-09-14T04:09:34.638 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:34.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:34.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:34.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:34.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:34.967 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:34.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:34.988 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:09:34.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-14T04:09:35.053 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:35.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:35.071 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:35.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:35.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:35.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:35.336 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:35.359 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:35.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:35.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:35.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:35.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:35 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:09:35.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:35 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:09:35.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:35.497 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:35.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:35.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 67787 67771 108.584 74.25 0.0147909 0.00920407 2024-09-14T04:09:35.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:35.744 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:35.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:35.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:35.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:35.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:35.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:35.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:36.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:36.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:36.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:36.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:36.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:36.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:36.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:36.465 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:36 smithi001 ceph-mon[107399]: pgmap v680: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:09:36.465 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:36 smithi001 ceph-mon[112165]: pgmap v680: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:09:36.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:36.556 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:09:36.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:36 smithi012 ceph-mon[101180]: pgmap v680: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:09:36.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:36.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:09:36.624344+0000 min lat: 0.00110309 max lat: 2.05165 avg lat: 0.00925279 2024-09-14T04:09:36.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:09:36.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 69084 69068 107.896 81.0625 0.00298814 0.00925279 2024-09-14T04:09:36.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:36] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:09:36.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:36.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:36.864 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:36.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:36.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:37.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:37.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:37.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:37.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:37.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:37.280 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:37.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:37.431 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:37.454 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:37.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:37.592 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:37.624 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 70021 70005 106.692 58.5625 0.00953266 0.00936777 2024-09-14T04:09:37.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:37.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:37.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:37.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:37.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:37.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: pgmap v681: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: pgmap v681: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:09:37.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:37.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:37.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:37.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:38.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:38.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:38.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: pgmap v681: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:09:38.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:38.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:38.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:38.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:38.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:38.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:38.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:38.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:38.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:38.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:38.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:38.625 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 71364 71348 106.151 83.9375 0.008757 0.00941588 2024-09-14T04:09:38.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:38.725 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:38.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:38.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:38.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:38.963 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:38.998 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:38.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:39.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:39.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:39.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:39.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:39.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:39.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:39.506 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:39.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:39.626 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 72787 72771 105.747 88.9375 0.0117107 0.00945131 2024-09-14T04:09:39.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:39.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:39.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:39.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:39.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:39.932 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:39.932 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:39.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:40.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:40.025 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:09:40.026 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:09:40.060 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:09:40.060 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:09:40.099 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:09:40.099 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:09:40.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:40.135 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:09:40.136 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:09:40.165 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:40.170 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:09:40.170 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:09:40.207 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:09:40.207 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:09:40.245 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:09:40.246 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:09:40.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:40.285 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:09:40.285 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:09:40.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:40.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:40.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:40.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:40 smithi001 ceph-mon[112165]: pgmap v682: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T04:09:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2746678337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:40 smithi001 ceph-mon[107399]: pgmap v682: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T04:09:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2746678337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:40.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:40.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:40 smithi012 ceph-mon[101180]: pgmap v682: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T04:09:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2746678337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:40.626 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 74052 74036 105.141 79.0625 0.0114164 0.00950608 2024-09-14T04:09:40.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:40.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:40.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:40.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:40.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:40.933 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:09:40.933 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:09:40.933 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:09:40.933 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:09:40.933 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:09:40.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:40.994 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:40.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:41.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:41.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:41.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:41.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:41.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:41.626 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 75313 75297 104.556 78.8125 0.00716085 0.00955877 2024-09-14T04:09:41.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:41.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:41.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:41.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:41.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:41.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:42.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:42.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:42.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:42.236 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:42.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:42.378 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:42 smithi001 ceph-mon[112165]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-14T04:09:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:42 smithi001 ceph-mon[107399]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-14T04:09:42.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:42.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:42 smithi012 ceph-mon[101180]: pgmap v683: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-14T04:09:42.621 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:42.626 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 76639 76623 104.084 82.875 0.0135945 0.00960287 2024-09-14T04:09:42.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:42.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:09:42.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:42.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:42.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:42.938 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:42.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:43.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:43.080 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:43.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:43.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:43.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:43.364 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:43.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:43.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:43.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:43.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:09:43.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:43.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 77994 77978 103.67 84.6875 0.0178763 0.00964066 2024-09-14T04:09:43.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:43.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:43.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:43.924 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:43.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:43.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:43.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:44.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:44.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:44.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:44.243 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:44.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:44.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:44 smithi001 ceph-mon[107399]: pgmap v684: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:09:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:44 smithi001 ceph-mon[112165]: pgmap v684: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:09:44.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:44.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:44 smithi012 ceph-mon[101180]: pgmap v684: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:09:44.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 79310 79294 103.223 82.25 0.00585516 0.00968303 2024-09-14T04:09:44.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:44.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:44.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:44.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:44.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:44.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:45.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:45.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:45.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:45.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:45.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:45.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:45.480 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:45.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:45.613 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:45.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 80643 80627 102.817 83.3125 0.00728475 0.00972108 2024-09-14T04:09:45.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:45.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:45.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:45.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:45.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:45.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:45.966 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:46.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:46.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:46.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:46.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.225 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:46.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:46.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:46.366 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:46.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:46.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:46 smithi001 ceph-mon[112165]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:09:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:46 smithi001 ceph-mon[107399]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:09:46.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:46 smithi012 ceph-mon[101180]: pgmap v685: 99 pgs: 99 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:09:46.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:46.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 81928 81912 102.366 80.3125 0.0114811 0.00976332 2024-09-14T04:09:46.659 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:46.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:46.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:46.803 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:46.834 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:46] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:09:46.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:46.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:46.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:47.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:47.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:47.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:47.216 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:47.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:47.354 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:47.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:47.456 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:47.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:47.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:47.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 83162 83146 101.871 77.125 0.0278357 0.00981186 2024-09-14T04:09:47.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:47.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:47.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:47.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:47.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:47.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:47.980 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:48.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.070 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:48.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:48.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:48.210 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:09:48.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:48.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:48.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:48.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:48.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:48 smithi001 ceph-mon[112165]: pgmap v686: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T04:09:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:48 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:09:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:48 smithi001 ceph-mon[107399]: pgmap v686: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T04:09:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:48 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:09:48.503 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:48 smithi012 ceph-mon[101180]: pgmap v686: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T04:09:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:48 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:09:48.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:48.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 84391 84375 101.389 76.8125 0.00890466 0.00985718 2024-09-14T04:09:48.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:48.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:48.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:48.872 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:48.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:48.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:49.012 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:49.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:49.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:49.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:49.213 INFO:teuthology.orchestra.run.smithi001.stdout:moved 25 / 297 (8.41751%) 2024-09-14T04:09:49.213 INFO:teuthology.orchestra.run.smithi001.stdout:avg 37.125 2024-09-14T04:09:49.213 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 9.35999 -> 9.38666 (expected baseline 5.69951) 2024-09-14T04:09:49.213 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.0 with 20 -> 16 pgs (0.538721 -> 0.430976 * mean) 2024-09-14T04:09:49.213 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.1 with 45 -> 47 pgs (1.21212 -> 1.26599 * mean) 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 1 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0271 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0298 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 weight 1.0000 -> 0.7000 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 weight 0.7367 -> 1.0000 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0 weight 0.4363 -> 0.3481 2024-09-14T04:09:49.214 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7 weight 0.8981 -> 1.0000 2024-09-14T04:09:49.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:49.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[112165]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"} : dispatch 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:09:49.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[107399]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T04:09:49.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"} : dispatch 2024-09-14T04:09:49.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:49.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:49.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:49.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:49 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:09:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2974794291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:09:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:49 smithi012 ceph-mon[101180]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T04:09:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"} : dispatch 2024-09-14T04:09:49.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 85230 85214 100.466 52.4375 0.00340025 0.00988099 2024-09-14T04:09:49.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:49.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:49.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:49.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:49.914 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:49.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:49.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:50.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:50.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:50.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.192 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:50.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:50.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:50.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:50.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: pgmap v688: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"}]': finished 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: pgmap v688: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"}]': finished 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:50.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:50.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: pgmap v688: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'22816','4':'45872','5':'65536','7':'65536'}"}]': finished 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:09:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:50.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 85230 85214 98.6053 0 - 0.00988099 2024-09-14T04:09:50.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:50.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:50.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:50.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:50.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:50.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:50.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:51.052 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:51.080 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:51.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:51.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:09:51.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:51.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:51.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:51.334 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:51.436 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:51.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:51.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:51 smithi001 ceph-mon[112165]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T04:09:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:51 smithi001 ceph-mon[112165]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T04:09:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:51 smithi001 ceph-mon[107399]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T04:09:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:51 smithi001 ceph-mon[107399]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T04:09:51.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:51 smithi012 ceph-mon[101180]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T04:09:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:51 smithi012 ceph-mon[101180]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T04:09:51.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:51.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 85363 85347 96.9638 4.15625 0.0162059 0.0103076 2024-09-14T04:09:51.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:51.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:51.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:51.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:51.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:51.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:52.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.026 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:52.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:52.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:52.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:52.302 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:52.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:52.440 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:52 smithi001 ceph-mon[112165]: pgmap v691: 99 pgs: 2 remapped+peering, 6 peering, 91 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 11 B/s, 0 objects/s recovering 2024-09-14T04:09:52.440 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:52 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:09:52.440 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:52 smithi001 ceph-mon[107399]: pgmap v691: 99 pgs: 2 remapped+peering, 6 peering, 91 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 11 B/s, 0 objects/s recovering 2024-09-14T04:09:52.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:52 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:09:52.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:52.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:52.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:52 smithi012 ceph-mon[101180]: pgmap v691: 99 pgs: 2 remapped+peering, 6 peering, 91 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 11 B/s, 0 objects/s recovering 2024-09-14T04:09:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:52 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:09:52.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 86328 86312 96.3093 60.3125 0.028321 0.0103781 2024-09-14T04:09:52.677 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:52.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:52.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:52.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:52.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:52.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:52.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:53.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:53.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:53.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:53.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:53.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:53.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:53 smithi001 ceph-mon[107399]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T04:09:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:53 smithi001 ceph-mon[112165]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T04:09:53.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:53.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:53 smithi012 ceph-mon[101180]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T04:09:53.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 87267 87251 95.6492 58.6875 0.0237136 0.0104488 2024-09-14T04:09:53.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:53.662 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:09:53.662 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:09:53.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:09:53.662 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:09:53.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249045 8.7 8:e36474f6:::benchmark_data_smithi001_125658_object123806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 521.31163798399996, 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0581002930000001, 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249045 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.663 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344301+0000", 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344303+0000", 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4959999999999999e-06 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344307+0000", 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0400000000000003e-06 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344317+0000", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5759999999999995e-06 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344324+0000", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7819999999999994e-06 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344858+0000", 2024-09-14T04:09:53.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00053357000000000003 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.344868+0000", 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6439999999999995e-06 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376162+0000", 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0312939839999999 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376249+0000", 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7023999999999999e-05 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.376790+0000", 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054109600000000002 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387731+0000", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010941683000000001 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402294+0000", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014562354 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402334+0000", 2024-09-14T04:09:53.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0154000000000002e-05 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402368+0000", 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.417e-05 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402402+0000", 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3720000000000002e-05 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.670 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.34299.0:249083 8.7 8:e6756ecc:::benchmark_data_smithi001_125658_object123844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 521.26999606599998, 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016715622, 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.671 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.34299", 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2656434194", 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 249083 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385943+0000", 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385944+0000", 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5899999999999996e-07 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385947+0000", 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.349e-06 2024-09-14T04:09:53.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385959+0000", 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1462e-05 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.385965+0000", 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1870000000000002e-06 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386615+0000", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00064992000000000001 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:12.386640+0000", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5106e-05 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387765+0000", 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0011247430000001 2024-09-14T04:09:53.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387888+0000", 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123702 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:09:53.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.387975+0000", 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6299999999999997e-05 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.394473+0000", 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064983289999999997 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402469+0000", 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079956669999999997 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402562+0000", 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3187000000000001e-05 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402616+0000", 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4555999999999997e-05 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:01:13.402659+0000", 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2555000000000002e-05 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161424 9.4 9:2dc0caff:::benchmark_data_smithi001_156540_object29830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 211.571878717, 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.134812339, 2024-09-14T04:09:53.679 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161424 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084061+0000", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084063+0000", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.86e-06 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084068+0000", 2024-09-14T04:09:53.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3700000000000003e-06 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084082+0000", 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.438e-05 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.084090+0000", 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9039999999999992e-06 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.096994+0000", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012904186999999999 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.097111+0000", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011634200000000001 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.097242+0000", 2024-09-14T04:09:53.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131612 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.107203+0000", 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099605730000000003 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.116776+0000", 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0095725740000000004 2024-09-14T04:09:53.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.119677+0000", 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029018899999999999 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.138479+0000", 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018802003000000001 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.138492+0000", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2500999999999999e-05 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.218791+0000", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0802988039999999 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.218873+0000", 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2338999999999999e-05 2024-09-14T04:09:53.686 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161431 9.4 9:2924b977:::benchmark_data_smithi001_156540_object29837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 211.54336365899999, 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.120078919, 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.687 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161431 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112576+0000", 2024-09-14T04:09:53.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112578+0000", 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.922e-06 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112594+0000", 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6348e-05 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112615+0000", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0559000000000001e-05 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112626+0000", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1138000000000001e-05 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112820+0000", 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000194589 2024-09-14T04:09:53.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.112965+0000", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014421099999999999 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.113115+0000", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015039900000000001 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.117389+0000", 2024-09-14T04:09:53.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042743069999999998 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139019+0000", 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021629598 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139027+0000", 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3319999999999992e-06 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219140+0000", 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.080112307 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219255+0000", 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011523099999999999 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232578+0000", 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013323583999999999 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232655+0000", 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6394000000000001e-05 2024-09-14T04:09:53.695 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:161439 9.4 9:21cf609a:::benchmark_data_smithi001_156540_object29845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 211.52548017999999, 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1061727830000001, 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.696 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 161439 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130459+0000", 2024-09-14T04:09:53.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130461+0000", 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5149999999999999e-06 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130465+0000", 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3980000000000001e-06 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130477+0000", 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2037e-05 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.130487+0000", 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3989999999999993e-06 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139087+0000", 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0086001710000000002 2024-09-14T04:09:53.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:22.139093+0000", 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5880000000000003e-06 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219287+0000", 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0801940729999999 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219394+0000", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106968 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.219524+0000", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000129252 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.228456+0000", 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089321609999999992 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.232699+0000", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042436389999999996 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236453+0000", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037530580000000001 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236516+0000", 2024-09-14T04:09:53.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3195999999999996e-05 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:23.236632+0000", 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116328 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.44714.0:178538 9.4 9:2947cf7b:::benchmark_data_smithi001_156540_object46944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:09:53.703 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 188.30174977600001, 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.014113509, 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.44714", 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 178538 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:09:53.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354190+0000", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354191+0000", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.238e-06 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354196+0000", 2024-09-14T04:09:53.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5809999999999998e-06 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354210+0000", 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4250000000000001e-05 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354217+0000", 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3880000000000002e-06 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354266+0000", 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8795000000000003e-05 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:45.354275+0000", 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8089999999999999e-06 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364693+0000", 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0104177759999999 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.707 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364776+0000", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3070000000000003e-05 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.364860+0000", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4460000000000001e-05 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.367445+0000", 2024-09-14T04:09:53.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025850030000000002 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.367905+0000", 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00046023199999999998 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368227+0000", 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032120800000000002 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368258+0000", 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1029999999999999e-05 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:06:46.368303+0000", 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5668999999999999e-05 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:22401 10.b e198/184 10:daef492a:::benchmark_data_smithi001_190881_object22400:head v 198'1456, mlcod=197'1454)", 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:09.964876+0000", 2024-09-14T04:09:53.710 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 43.691063862999997, 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.113012323, 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.964876+0000", 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.711 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.964876+0000", 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.964876+0000", 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1699999999999999e-07 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.964934+0000", 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8254000000000002e-05 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074954+0000", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.110019987 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074961+0000", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3159999999999999e-06 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.075151+0000", 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018923900000000001 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.075169+0000", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8258999999999999e-05 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.077811+0000", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026415420000000002 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.077888+0000", 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7409000000000004e-05 2024-09-14T04:09:53.714 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:22410 10.2 10:434aeb9d:::benchmark_data_smithi001_190881_object22409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:09.969572+0000", 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 43.686367816000001, 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.108707443, 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.715 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 22410 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.969572+0000", 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.969572+0000", 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.969574+0000", 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3920000000000001e-06 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.969605+0000", 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0964999999999999e-05 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074822+0000", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10521676200000001 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074840+0000", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8127000000000001e-05 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.075007+0000", 2024-09-14T04:09:53.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016680899999999999 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.075160+0000", 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015325399999999999 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.075309+0000", 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000149044 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.719 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.077066+0000", 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00175717 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078031+0000", 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00096451299999999996 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078066+0000", 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5624000000000003e-05 2024-09-14T04:09:53.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078226+0000", 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015963899999999999 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078279+0000", 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3143999999999998e-05 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:22411 10.4 e198/196 10:281e4e94:::benchmark_data_smithi001_190881_object22410:head v 198'1378, mlcod=197'1376)", 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:09.970255+0000", 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 43.685683990999998, 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.108645766, 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.970255+0000", 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.970255+0000", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.970256+0000", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6e-07 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:09.970331+0000", 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5247999999999995e-05 2024-09-14T04:09:53.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074745+0000", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10441362799999999 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074755+0000", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0581000000000001e-05 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074875+0000", 2024-09-14T04:09:53.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119454 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.074901+0000", 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6533000000000002e-05 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078850+0000", 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039492440000000002 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:10.078901+0000", 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0818e-05 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:44404 10.2 10:479c5581:::benchmark_data_smithi001_190881_object44403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:20.771978+0000", 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.883961823, 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.086261813000000007, 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.726 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 44404 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.771978+0000", 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.771978+0000", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.771979+0000", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5880000000000001e-06 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772005+0000", 2024-09-14T04:09:53.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5310999999999999e-05 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772017+0000", 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2897e-05 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772025+0000", 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7889999999999995e-06 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772131+0000", 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105812 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772262+0000", 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013045099999999999 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772476+0000", 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021448600000000001 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.854084+0000", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.081608196999999993 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.854128+0000", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3986999999999997e-05 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.857920+0000", 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003791446 2024-09-14T04:09:53.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.858125+0000", 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020561099999999999 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.858239+0000", 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114238 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.732 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:44401 10.b e198/184 10:dd75f63c:::benchmark_data_smithi001_190881_object44400:head v 198'2804, mlcod=198'2802)", 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:20.772010+0000", 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.883929121999998, 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080660924999999994, 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772010+0000", 2024-09-14T04:09:53.733 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772010+0000", 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772011+0000", 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5799999999999998e-07 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772059+0000", 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7815999999999998e-05 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772070+0000", 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1459e-05 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772077+0000", 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4509999999999997e-06 2024-09-14T04:09:53.735 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772145+0000", 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7953000000000005e-05 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.772163+0000", 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8295999999999999e-05 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.736 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852627+0000", 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080463483000000002 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852671+0000", 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4608999999999999e-05 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.737 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:44403 10.b e198/184 10:d862d703:::benchmark_data_smithi001_190881_object44402:head v 198'2805, mlcod=198'2802)", 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:20.774394+0000", 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.881544992000002, 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.081112279999999995, 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.738 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.739 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:53.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:53.740 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.740 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.740 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.774394+0000", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.774394+0000", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.774396+0000", 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.124e-06 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.774448+0000", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.253e-05 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852621+0000", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.078172631000000006 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852632+0000", 2024-09-14T04:09:53.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1533e-05 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852699+0000", 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6772000000000004e-05 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.852718+0000", 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9154999999999999e-05 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.855448+0000", 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027295190000000001 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:20.855507+0000", 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9015999999999999e-05 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.744 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85146 10.2 10:4e8e28f3:::benchmark_data_smithi001_190881_object85145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:09:53.746 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.4487661230000004, 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1709060940000002, 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85146 2024-09-14T04:09:53.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:09:53.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207175+0000", 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.813e-06 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207199+0000", 2024-09-14T04:09:53.749 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4312999999999999e-05 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207216+0000", 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6694999999999999e-05 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207223+0000", 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.08e-06 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207276+0000", 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3038999999999998e-05 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207427+0000", 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015100000000000001 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207565+0000", 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013732699999999999 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223766+0000", 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016201706 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223775+0000", 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5890000000000003e-06 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214581+0000", 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99080585499999996 2024-09-14T04:09:53.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214598+0000", 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7051e-05 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378012+0000", 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1634141140000001 2024-09-14T04:09:53.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378079+0000", 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7512e-05 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.755 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85153 10.2 10:4ace6224:::benchmark_data_smithi001_190881_object85152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.4343556839999998, 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1713794549999998, 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.756 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85153 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.757 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221586+0000", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9290000000000001e-06 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221614+0000", 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8419999999999999e-05 2024-09-14T04:09:53.758 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221629+0000", 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4602e-05 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221638+0000", 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8959999999999992e-06 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.759 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223951+0000", 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023134240000000001 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223958+0000", 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.754e-06 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214629+0000", 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.990671623 2024-09-14T04:09:53.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214640+0000", 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.049e-05 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379336+0000", 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1646960749999999 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379484+0000", 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014773 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379590+0000", 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106259 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384020+0000", 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044296270000000002 2024-09-14T04:09:53.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384076+0000", 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6487999999999999e-05 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390537+0000", 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064613240000000001 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390591+0000", 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3164999999999997e-05 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.392963+0000", 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023726490000000001 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85211 10.2 10:452ef993:::benchmark_data_smithi001_190881_object85210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.3777396470000003, 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1165732400000001, 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85211 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:09:53.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278202+0000", 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9640000000000002e-06 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278222+0000", 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9796000000000002e-05 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278234+0000", 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1922999999999999e-05 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278242+0000", 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3000000000000002e-06 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278320+0000", 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8647999999999996e-05 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278333+0000", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.257e-05 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214669+0000", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.936335849 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214677+0000", 2024-09-14T04:09:53.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1559999999999995e-06 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381729+0000", 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.167051839 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381867+0000", 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013809400000000001 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381978+0000", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011068 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384104+0000", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002125886 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.386784+0000", 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002680038 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393032+0000", 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062480950000000004 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393073+0000", 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1409999999999998e-05 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.394773+0000", 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001699992 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85279 10.2 10:4d1935a1:::benchmark_data_smithi001_190881_object85278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e203)", 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:51.504610+0000", 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.1513294539999999, 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.086437892000000002, 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.773 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85279 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.504610+0000", 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.774 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.504610+0000", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.504611+0000", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2550000000000001e-06 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.504636+0000", 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4451999999999999e-05 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.556557+0000", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.051921512000000003 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.556576+0000", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8516999999999999e-05 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.573782+0000", 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017206445000000001 2024-09-14T04:09:53.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.573901+0000", 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118689 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.574014+0000", 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011322100000000001 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.579474+0000", 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0054602670000000004 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.588070+0000", 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0085957599999999992 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.588109+0000", 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8689000000000002e-05 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.588160+0000", 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.092e-05 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.591048+0000", 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028881649999999998 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:86874 10.2 10:467a3627:::benchmark_data_smithi001_190881_object86873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-14T04:09:53.779 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:53.196920+0000", 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.45901978100000002, 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.100643144, 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 86874 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196920+0000", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196920+0000", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196921+0000", 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6929999999999999e-06 2024-09-14T04:09:53.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196945+0000", 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4051999999999999e-05 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196958+0000", 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2081e-05 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196965+0000", 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7649999999999993e-06 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.217759+0000", 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020793615000000001 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.218002+0000", 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024311299999999999 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.218226+0000", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000223816 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.231896+0000", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01366967 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.231916+0000", 2024-09-14T04:09:53.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0511999999999999e-05 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.291878+0000", 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.059961949 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.292006+0000", 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000128164 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.297563+0000", 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0055567139999999999 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:86875 10.2 10:495e3fb9:::benchmark_data_smithi001_190881_object86874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:53.196980+0000", 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.45895909600000001, 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.104102526, 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.786 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 86875 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196980+0000", 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196980+0000", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.196981+0000", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6100000000000001e-07 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.197007+0000", 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6126e-05 2024-09-14T04:09:53.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.197015+0000", 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1610000000000003e-06 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.197021+0000", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3929999999999999e-06 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.221220+0000", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024199818000000001 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.221390+0000", 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000169784 2024-09-14T04:09:53.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.221557+0000", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001671 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.231921+0000", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01036397 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.243132+0000", 2024-09-14T04:09:53.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011210688999999999 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.297688+0000", 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.054555665000000003 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.297753+0000", 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5582999999999999e-05 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.301083+0000", 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033296760000000002 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:86887 10.2 10:43d0f649:::benchmark_data_smithi001_190881_object86886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:53.213612+0000", 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.44232763000000003, 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.090786215000000003, 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.793 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 86887 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213612+0000", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213612+0000", 2024-09-14T04:09:53.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213614+0000", 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1299999999999999e-06 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213641+0000", 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6923999999999998e-05 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213659+0000", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8046999999999998e-05 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.213670+0000", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0896999999999999e-05 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.224585+0000", 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010915622999999999 2024-09-14T04:09:53.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.224735+0000", 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000150001 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.224911+0000", 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000175728 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.231927+0000", 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007016034 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.254754+0000", 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022826641000000002 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.301241+0000", 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.046486760000000002 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.301369+0000", 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000128749 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.304398+0000", 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0030286810000000001 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:86894 10.2 10:4e1daadd:::benchmark_data_smithi001_190881_object86893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2024-09-14T04:09:53.799 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:53.216866+0000", 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.43907359499999998, 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.092003529000000001, 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 86894 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216866+0000", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216866+0000", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216868+0000", 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8649999999999999e-06 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216894+0000", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.618e-05 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216910+0000", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6426000000000002e-05 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.216919+0000", 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6009999999999995e-06 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.227966+0000", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011046719999999999 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.228316+0000", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000349876 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.228841+0000", 2024-09-14T04:09:53.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000525553 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.254952+0000", 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.026110511999999999 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.255064+0000", 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000112246 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.305014+0000", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.049950382000000002 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.305080+0000", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5470999999999997e-05 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:53.308869+0000", 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037896969999999999 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.805 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.806 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.806 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:09:53.806 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:09:53.806 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:09:53.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:53.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:53.889 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:53.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:53.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:54.029 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:54.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:54.171 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:54.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:54.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:54.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:54.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:54.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:54.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:54 smithi012 ceph-mon[101180]: pgmap v694: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5310/256005 objects misplaced (2.074%); 256 KiB/s, 4 objects/s recovering 2024-09-14T04:09:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:54 smithi012 ceph-mon[101180]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T04:09:54.628 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 88208 88192 95.0141 58.8125 0.0111516 0.0105185 2024-09-14T04:09:54.658 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:54 smithi001 ceph-mon[107399]: pgmap v694: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5310/256005 objects misplaced (2.074%); 256 KiB/s, 4 objects/s recovering 2024-09-14T04:09:54.658 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:54 smithi001 ceph-mon[107399]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T04:09:54.658 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:54 smithi001 ceph-mon[112165]: pgmap v694: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5310/256005 objects misplaced (2.074%); 256 KiB/s, 4 objects/s recovering 2024-09-14T04:09:54.658 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:54 smithi001 ceph-mon[112165]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T04:09:54.693 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:09:54.693 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:09:54.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:54.700 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:09:54.700 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:09:54.705 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:54.744 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:09:54.744 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:09:54.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:54.780 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:09:54.780 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:09:54.814 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:09:54.815 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:09:54.846 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:09:54.846 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:09:54.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:54.874 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:09:54.874 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:09:54.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:54.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:54.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:54.906 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:09:54.906 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:09:54.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:55.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:55.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:55.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:55.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:55.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:09:55.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:55.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:55.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:55.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:55.561 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1196157613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:55.561 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1196157613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:55.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1196157613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:09:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:09:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=cleanup t=2024-09-14T04:09:55.277669146Z level=info msg="Completed cleanup jobs" duration=27.721253ms 2024-09-14T04:09:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:09:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugins.update.checker t=2024-09-14T04:09:55.420902428Z level=info msg="Update check succeeded" duration=62.065691ms 2024-09-14T04:09:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:09:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana.update.checker t=2024-09-14T04:09:55.428090694Z level=info msg="Update check succeeded" duration=61.951216ms 2024-09-14T04:09:55.602 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:09:55.603 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:09:55.603 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:09:55.603 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2024-09-14T04:09:55.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-14T04:09:55.631 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 89108 89092 94.3531 56.25 0.00911945 0.010591 2024-09-14T04:09:55.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:55.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:55.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:55.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:55.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:55.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:55.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:56.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:56.124 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:56.226 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:56.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:56.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:56.454 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:56.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:56.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:56 smithi012 ceph-mon[101180]: pgmap v696: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 94 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 10469/257118 objects misplaced (4.072%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:09:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:56 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:09:56.631 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:09:56.630928+0000 min lat: 0.00110309 max lat: 2.24751 avg lat: 0.0106626 2024-09-14T04:09:56.631 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:09:56.631 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 90032 90016 93.743 57.75 0.0142366 0.0106626 2024-09-14T04:09:56.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:56 smithi001 ceph-mon[107399]: pgmap v696: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 94 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 10469/257118 objects misplaced (4.072%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:09:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:56 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:09:56.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:56 smithi001 ceph-mon[112165]: pgmap v696: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 remapped+peering, 94 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 10469/257118 objects misplaced (4.072%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:09:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:56 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:09:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.735 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:56.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:56.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:56.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:56.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:09:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:09:56] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-09-14T04:09:56.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:57.016 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:57.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:57.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:57.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:57.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:57.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:57.458 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:57.493 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:57.551 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:09:57.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:57 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 10713/265452 objects degraded (4.036%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:57.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:57.632 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 90948 90932 93.1432 57.25 0.0123441 0.0107304 2024-09-14T04:09:57.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:57 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 10713/265452 objects degraded (4.036%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:57.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:57 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 10713/265452 objects degraded (4.036%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:09:57.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:57.761 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:57.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:57.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:57.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:58.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:58.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:58.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:58.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:09:58.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:09:58.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:09:58.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:09:58.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:09:58.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:09:58 smithi012 ceph-mon[101180]: pgmap v697: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 10713/265452 objects degraded (4.036%); 10469/265452 objects misplaced (3.944%); 1013 KiB/s, 14 objects/s recovering 2024-09-14T04:09:58.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:58.632 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 91912 91896 92.6127 60.25 0.0147724 0.0107917 2024-09-14T04:09:58.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:09:58 smithi001 ceph-mon[107399]: pgmap v697: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 10713/265452 objects degraded (4.036%); 10469/265452 objects misplaced (3.944%); 1013 KiB/s, 14 objects/s recovering 2024-09-14T04:09:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:09:58 smithi001 ceph-mon[112165]: pgmap v697: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 10713/265452 objects degraded (4.036%); 10469/265452 objects misplaced (3.944%); 1013 KiB/s, 14 objects/s recovering 2024-09-14T04:09:58.746 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:58.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:09:58.859 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:09:58.882 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:58.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:09:58.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:59.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:59.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:09:59.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:09:59.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:09:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:59.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:09:59.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.478 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:59.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:59.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:09:59.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:09:59.632 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 92761 92745 91.9849 53.0625 0.0560735 0.0108641 2024-09-14T04:09:59.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:09:59.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:09:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:09:59.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:09:59.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:09:59.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:09:59.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:00.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:00.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:00.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:00.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:00.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:00.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:00.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:00.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:00.423 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:10:00.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:00.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: pgmap v698: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 10713/268935 objects degraded (3.983%); 10367/268935 objects misplaced (3.855%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:10:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2594876699' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:10:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: [WRN] PG_DEGRADED: Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:10:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:00 smithi012 ceph-mon[101180]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,3] 2024-09-14T04:10:00.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:00.632 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 15 93701 93686 91.4666 58.8125 0.00923307 0.0109273 2024-09-14T04:10:00.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: pgmap v698: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 10713/268935 objects degraded (3.983%); 10367/268935 objects misplaced (3.855%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2594876699' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: [WRN] PG_DEGRADED: Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[112165]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,3] 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: pgmap v698: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 656 op/s; 10713/268935 objects degraded (3.983%); 10367/268935 objects misplaced (3.855%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2594876699' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: [WRN] PG_DEGRADED: Degraded data redundancy: 10713/268935 objects degraded (3.983%), 2 pgs degraded 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: pg 10.0 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:10:00.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:00 smithi001 ceph-mon[107399]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,3] 2024-09-14T04:10:00.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:00.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:00.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:00.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:00.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:01.036 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:01.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:01.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:01.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:01.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:01.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:01.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:01.456 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:01.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:01.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[112165]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T04:10:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:01.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:01 smithi001 ceph-mon[107399]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T04:10:01.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:01 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:01 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:01 smithi012 ceph-mon[101180]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T04:10:01.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:01.633 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 94615 94599 90.9364 57.0625 0.00821787 0.010991 2024-09-14T04:10:01.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:01.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:01.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:01.899 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:01.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:02.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:02.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:02.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:02.176 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:02.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:02.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:02.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:02.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:02.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:02.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:02.601 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:02.633 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 95546 95530 90.4402 58.1875 0.0133936 0.0110507 2024-09-14T04:10:02.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:02.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:02 smithi001 ceph-mon[107399]: pgmap v700: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 10713/278319 objects degraded (3.849%); 10234/278319 objects misplaced (3.677%); 2.5 MiB/s, 38 objects/s recovering 2024-09-14T04:10:02.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:02 smithi001 ceph-mon[112165]: pgmap v700: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 10713/278319 objects degraded (3.849%); 10234/278319 objects misplaced (3.677%); 2.5 MiB/s, 38 objects/s recovering 2024-09-14T04:10:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:02.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:02 smithi012 ceph-mon[101180]: pgmap v700: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 10713/278319 objects degraded (3.849%); 10234/278319 objects misplaced (3.677%); 2.5 MiB/s, 38 objects/s recovering 2024-09-14T04:10:02.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:02.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:02.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:02.994 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:03.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:03.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:03.180 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:03.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:03.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:03.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:03.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:03.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:03.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:03.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:03.633 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 96460 96444 89.943 57.125 0.00654878 0.0111125 2024-09-14T04:10:03.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:03 smithi001 ceph-mon[107399]: pgmap v701: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 10713/281826 objects degraded (3.801%); 10124/281826 objects misplaced (3.592%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:10:03.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:03 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/281826 objects degraded (3.801%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:03.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:03 smithi001 ceph-mon[112165]: pgmap v701: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 10713/281826 objects degraded (3.801%); 10124/281826 objects misplaced (3.592%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:10:03.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:03 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/281826 objects degraded (3.801%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:03.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:03.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:03.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:10:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:03 smithi012 ceph-mon[101180]: pgmap v701: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 10713/281826 objects degraded (3.801%); 10124/281826 objects misplaced (3.592%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:10:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:03 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/281826 objects degraded (3.801%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:03.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:03.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:03.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:04.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:04.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:04.169 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:04.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:04.407 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:04.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:04.577 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.633 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 97285 97269 89.3786 51.5625 0.00810038 0.0111814 2024-09-14T04:10:04.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:04.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:04.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:04.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:04.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.002 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:05.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:05.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:05.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:05.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:05.287 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:05.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:05.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:05.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:05.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:05.633 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 98174 98158 88.8885 55.5625 0.00741017 0.0112441 2024-09-14T04:10:05.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:05.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:05.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:05.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:05.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:05.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:05.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:06.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:06.117 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:06.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[107399]: pgmap v702: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 10713/283536 objects degraded (3.778%); 10007/283536 objects misplaced (3.529%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:05 smithi001 ceph-mon[112165]: pgmap v702: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 10713/283536 objects degraded (3.778%); 10007/283536 objects misplaced (3.529%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:06.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:06.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:05 smithi012 ceph-mon[101180]: pgmap v702: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 10713/283536 objects degraded (3.778%); 10007/283536 objects misplaced (3.529%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:06.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:06.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:06.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:06.531 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:06.633 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:06.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 99005 98989 88.3606 51.9375 0.0171097 0.0113112 2024-09-14T04:10:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:06.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:06.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:06.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:06.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:06.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:06.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:06] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-14T04:10:07.045 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:07.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:07.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:07.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:07.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:07.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:07.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:07.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 99816 99800 87.83 50.6875 0.00842052 0.0113798 2024-09-14T04:10:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.635 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:07.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:07.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:07.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:07.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:07.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:08.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:08.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:08.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:08.192 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:08.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:08.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:08.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:08.431 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:08 smithi001 ceph-mon[107399]: pgmap v703: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 10713/292800 objects degraded (3.659%); 10007/292800 objects misplaced (3.418%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:10:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:08 smithi001 ceph-mon[112165]: pgmap v703: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 10713/292800 objects degraded (3.659%); 10007/292800 objects misplaced (3.418%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:10:08.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:08.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:08.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:08 smithi012 ceph-mon[101180]: pgmap v703: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 10713/292800 objects degraded (3.659%); 10007/292800 objects misplaced (3.418%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:08.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:10:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:08.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:08.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 100595 100579 87.2864 48.6875 0.0123224 0.0114511 2024-09-14T04:10:08.650 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-14T04:10:08.713 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:08.751 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:08.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:08.888 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:08.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:08.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.028 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:09.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:09.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:09.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:09.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:09.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:09.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:09.432 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[107399]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/292800 objects degraded (3.659%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[112165]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T04:10:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/292800 objects degraded (3.659%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:09.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:09.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3233954210' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:10:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:09 smithi012 ceph-mon[101180]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T04:10:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/292800 objects degraded (3.659%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:09.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 15 101380 101365 86.7636 49.125 0.0204705 0.0115192 2024-09-14T04:10:09.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:09.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:09.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:09.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:09.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:09.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:09.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:09.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:09.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:10.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:10.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:10.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:10.269 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:10.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:10.406 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:10.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:10.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:10 smithi012 ceph-mon[101180]: pgmap v705: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s; 10713/295956 objects degraded (3.620%); 9882/295956 objects misplaced (3.339%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:10:10.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 102251 102235 86.3259 54.375 0.0107117 0.0115781 2024-09-14T04:10:10.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:10.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:10 smithi001 ceph-mon[107399]: pgmap v705: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s; 10713/295956 objects degraded (3.620%); 9882/295956 objects misplaced (3.339%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:10:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:10 smithi001 ceph-mon[112165]: pgmap v705: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s; 10713/295956 objects degraded (3.620%); 9882/295956 objects misplaced (3.339%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:10:10.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:10.811 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:10.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:10.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:10.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:11.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:11.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:11.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.217 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:11.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:11.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:11.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:11.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:11.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:11.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:11.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 103065 103049 85.8532 50.875 0.0483089 0.0116413 2024-09-14T04:10:11.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:11.739 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:11.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:11.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:11.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:11.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:12.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:12.053 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:12.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.199 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:12.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:12.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:12.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:12.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:12.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:12.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:12 smithi012 ceph-mon[101180]: pgmap v706: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 10713/304137 objects degraded (3.522%); 9736/304137 objects misplaced (3.201%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:12.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:12.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 103822 103806 85.3461 47.3125 0.032815 0.0117111 2024-09-14T04:10:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:12 smithi001 ceph-mon[107399]: pgmap v706: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 10713/304137 objects degraded (3.522%); 9736/304137 objects misplaced (3.201%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:12 smithi001 ceph-mon[112165]: pgmap v706: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 10713/304137 objects degraded (3.522%); 9736/304137 objects misplaced (3.201%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:12.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:12.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:12.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:12.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:12.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:12.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:13.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:13.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:13.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:13.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:13.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:13.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:13.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:13.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:13.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:13.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:13.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 104623 104607 84.8879 50.0625 0.0180672 0.0117738 2024-09-14T04:10:13.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:13.752 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:13.853 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:13.926 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:13.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:14.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:14.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:14.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:14.202 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:14.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:14.343 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:14.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:14.484 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:10:14.484 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:10:14.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:14.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:14 smithi012 ceph-mon[101180]: pgmap v707: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 10713/307305 objects degraded (3.486%); 9636/307305 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:10:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/307305 objects degraded (3.486%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:14.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:14.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:14.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 105471 105455 84.4791 53 0.0621744 0.0118302 2024-09-14T04:10:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:14 smithi001 ceph-mon[107399]: pgmap v707: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 10713/307305 objects degraded (3.486%); 9636/307305 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:10:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/307305 objects degraded (3.486%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:14.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:14 smithi001 ceph-mon[112165]: pgmap v707: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 10713/307305 objects degraded (3.486%); 9636/307305 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:10:14.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/307305 objects degraded (3.486%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:14.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:14.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:14.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:14.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:14.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:15.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:15.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:15.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:15.185 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:10:15.197 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:15.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:15.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:15.345 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:15.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:15.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:15.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:15.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:15.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:15.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:15.635 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 106317 106301 84.0791 52.875 0.0110291 0.0118885 2024-09-14T04:10:15.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:15.768 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:15.870 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:15.910 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:15.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:16.004 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:10:16.005 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:10:16.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:16.040 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:10:16.040 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:10:16.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:16.076 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:10:16.076 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:10:16.118 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:10:16.118 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:10:16.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:16.164 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:10:16.164 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:10:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:16.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:16.216 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:10:16.217 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:10:16.249 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:10:16.249 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:10:16.279 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:10:16.279 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:10:16.280 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:16 smithi001 ceph-mon[112165]: pgmap v708: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 10713/308910 objects degraded (3.468%); 9513/308910 objects misplaced (3.080%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:16.280 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/357145947' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:16.280 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:16 smithi001 ceph-mon[107399]: pgmap v708: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 10713/308910 objects degraded (3.468%); 9513/308910 objects misplaced (3.080%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:16.280 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/357145947' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:16.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:16.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:16.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:16.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:16.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:16 smithi012 ceph-mon[101180]: pgmap v708: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 10713/308910 objects degraded (3.468%); 9513/308910 objects misplaced (3.080%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/357145947' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:16.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:16.634 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:10:16.634521+0000 min lat: 0.00110309 max lat: 2.24751 avg lat: 0.0119436 2024-09-14T04:10:16.635 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:10:16.635 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 107166 107150 83.6913 53.0625 0.0174151 0.0119436 2024-09-14T04:10:16.723 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:16.765 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:16.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:16.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:16.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:16.973 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:16] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-14T04:10:17.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:17.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:17.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:17.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:17.285 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.320 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:17.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:17.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:17.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:17.560 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:17.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:17.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:17.635 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 108047 108031 83.3379 55.0625 0.0123975 0.0119933 2024-09-14T04:10:17.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:17.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:17.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:17.876 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:17.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:17.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:18.021 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:18.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:18.123 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:18.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:18.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:18.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:18.361 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:18.404 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:18.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:18.532 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:18 smithi001 ceph-mon[112165]: pgmap v709: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10713/317406 objects degraded (3.375%); 9513/317406 objects misplaced (2.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:18.532 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:18 smithi001 ceph-mon[107399]: pgmap v709: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10713/317406 objects degraded (3.375%); 9513/317406 objects misplaced (2.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:18.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:18.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:18 smithi012 ceph-mon[101180]: pgmap v709: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10713/317406 objects degraded (3.375%); 9513/317406 objects misplaced (2.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:18.635 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 108909 108893 82.9786 53.875 0.0157285 0.0120455 2024-09-14T04:10:18.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:18.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:18.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:18.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:18.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:18.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:19.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:19.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:19.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:19.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:19.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:19.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:19.359 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:10:19.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:10:19.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:19.460 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:19.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/317406 objects degraded (3.375%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/317406 objects degraded (3.375%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:19.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/317406 objects degraded (3.375%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.605 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:19.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:19.637 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 109698 109682 82.5719 49.3125 0.0102419 0.012105 2024-09-14T04:10:19.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:19.747 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:19.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:19.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:19.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:19.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:20.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:20.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:20.169 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:20.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:20.274 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:10:20.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:20.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:10:20.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:20.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:20.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:20.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:20 smithi012 ceph-mon[101180]: pgmap v710: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 10713/320553 objects degraded (3.342%); 9399/320553 objects misplaced (2.932%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:10:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3303709671' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:20 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:20.636 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 15 110507 110492 82.1916 50.625 0.0158018 0.0121612 2024-09-14T04:10:20.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[107399]: pgmap v710: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 10713/320553 objects degraded (3.342%); 9399/320553 objects misplaced (2.932%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:10:20.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3303709671' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:20.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[112165]: pgmap v710: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 10713/320553 objects degraded (3.342%); 9399/320553 objects misplaced (2.932%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:10:20.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3303709671' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:10:20.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:20.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:20.737 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:20.838 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:20.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:20.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:20.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:21.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:21.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:21.188 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:21.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:21.320 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:21.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:21.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:21.549 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:21.549 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:10:21.549 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[107399]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T04:10:21.550 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:21.550 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:10:21.550 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:21 smithi001 ceph-mon[112165]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T04:10:21.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:21.564 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:21.566 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:21.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:21 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:10:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:21 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:10:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:21 smithi012 ceph-mon[101180]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T04:10:21.623 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:10:21.623 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:10:21.623 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:10:21.623 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:10:21.623 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:10:21.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 111324 111308 81.819 51 0.0214295 0.0122166 2024-09-14T04:10:21.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:21.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:21.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:21.867 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:21.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:21.969 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:22.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:22.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:22.146 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:22.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:22.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:22.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:22.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:22.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:22.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:22.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:22 smithi012 ceph-mon[101180]: pgmap v712: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 10713/329193 objects degraded (3.254%); 9278/329193 objects misplaced (2.818%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:10:22.615 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:22 smithi001 ceph-mon[107399]: pgmap v712: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 10713/329193 objects degraded (3.254%); 9278/329193 objects misplaced (2.818%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:10:22.615 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:22 smithi001 ceph-mon[112165]: pgmap v712: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 10713/329193 objects degraded (3.254%); 9278/329193 objects misplaced (2.818%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:10:22.642 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 112106 112090 81.436 48.875 0.00541651 0.0122731 2024-09-14T04:10:22.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:22.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:22.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:22.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:22.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:22.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:22.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:23.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:10:23.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:23.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:23.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:23.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:23.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:23.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:23.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.568 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:23.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:23.642 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 112832 112816 81.0215 45.375 0.00601329 0.0123346 2024-09-14T04:10:23.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:23.696 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:23.797 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:23.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:23.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:23.979 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:23.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:24.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:24.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:24.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:24.272 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:24.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:24.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:24.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:24.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:24 smithi012 ceph-mon[101180]: pgmap v713: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 10713/332307 objects degraded (3.224%); 9171/332307 objects misplaced (2.760%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/332307 objects degraded (3.224%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:24.642 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 113536 113520 80.6008 44 0.0221263 0.0123969 2024-09-14T04:10:24.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:24.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:24 smithi001 ceph-mon[107399]: pgmap v713: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 10713/332307 objects degraded (3.224%); 9171/332307 objects misplaced (2.760%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/332307 objects degraded (3.224%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:24 smithi001 ceph-mon[112165]: pgmap v713: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 10713/332307 objects degraded (3.224%); 9171/332307 objects misplaced (2.760%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/332307 objects degraded (3.224%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:24.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:24.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:24.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:24.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:24.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:24.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:25.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:25.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:25.199 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:25.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:25.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:25.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:25.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:25.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:25.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:25.518 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:25.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:25.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 114318 114302 80.2438 48.875 0.0120686 0.0124566 2024-09-14T04:10:25.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:25.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:25.797 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:25.899 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:25.936 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:25.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:26.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:26.072 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:26.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:26.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:26.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:26.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:26.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:26.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:26.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:26.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:26 smithi012 ceph-mon[101180]: pgmap v714: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10713/333732 objects degraded (3.210%); 9044/333732 objects misplaced (2.710%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:26.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.636 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:26.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0181 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 115137 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 79.9402 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 43.7558 2024-09-14T04:10:26.643 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 234 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1279 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 700.103 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3744 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0125038 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.0426242 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 2.24751 2024-09-14T04:10:26.644 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00110309 2024-09-14T04:10:26.649 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:10:26.708 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:26 smithi001 ceph-mon[107399]: pgmap v714: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10713/333732 objects degraded (3.210%); 9044/333732 objects misplaced (2.710%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:26.708 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:26 smithi001 ceph-mon[112165]: pgmap v714: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10713/333732 objects degraded (3.210%); 9044/333732 objects misplaced (2.710%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:26.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:26.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:26.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:26.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:26.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:26] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:10:27.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:27.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:27.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:27.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:27.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:27.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:27.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.456 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:27.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:27.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:27.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:27.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:27.724 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:27.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:27.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:27.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:27.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:27.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:28.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:28.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:10:28.139 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:28.241 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.281 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:28.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:28.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:28.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:28.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:28.563 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:28 smithi012 ceph-mon[101180]: pgmap v715: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 10713/341613 objects degraded (3.136%); 9044/341613 objects misplaced (2.647%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:28.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:28 smithi001 ceph-mon[107399]: pgmap v715: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 10713/341613 objects degraded (3.136%); 9044/341613 objects misplaced (2.647%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:28.657 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:28 smithi001 ceph-mon[112165]: pgmap v715: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 10713/341613 objects degraded (3.136%); 9044/341613 objects misplaced (2.647%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:28.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:28.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:28.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:28.840 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:28.942 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:28.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:28.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:29.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:29.111 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:29.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.248 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:29.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:29.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:29.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:29.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:29.503 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/341613 objects degraded (3.136%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:29.503 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/341613 objects degraded (3.136%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:29.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/341613 objects degraded (3.136%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:29.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:29.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:29.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:29.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:29.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:29.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:29.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:29.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:30.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.088 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:30.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:30.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:30.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:30.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:30.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:30.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:30.437 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:10:30.438 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:10:30.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:30.480 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:10:30.481 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:10:30.512 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:30.525 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:30 smithi001 ceph-mon[107399]: pgmap v716: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 7.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 800 op/s; 10713/343122 objects degraded (3.122%); 8936/343122 objects misplaced (2.604%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:30.525 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:30 smithi001 ceph-mon[112165]: pgmap v716: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 7.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 800 op/s; 10713/343122 objects degraded (3.122%); 8936/343122 objects misplaced (2.604%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:30.526 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:10:30.526 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:10:30.559 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:10:30.559 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:10:30.593 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:10:30.594 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:10:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:30 smithi012 ceph-mon[101180]: pgmap v716: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 7.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 800 op/s; 10713/343122 objects degraded (3.122%); 8936/343122 objects misplaced (2.604%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:10:30.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:30.627 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:10:30.628 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:10:30.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:30.656 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:10:30.656 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:10:30.698 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:10:30.698 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:10:30.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:30.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:30.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:30.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:30.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:30.988 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:31.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:31.050 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:31.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:31.187 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:31.283 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:31.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.321 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:31.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:31.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:31.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:31.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:31.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4088402913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:31.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:31.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:31.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4088402913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4088402913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:31.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:31.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:31.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:31.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:31.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:32.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:32.125 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:32.168 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:32.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:32.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:32.415 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:32 smithi001 ceph-mon[107399]: pgmap v717: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 94 B/s rd, 46 MiB/s wr, 977 op/s; 10713/337779 objects degraded (3.172%); 8786/337779 objects misplaced (2.601%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:10:32.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:32 smithi001 ceph-mon[112165]: pgmap v717: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 94 B/s rd, 46 MiB/s wr, 977 op/s; 10713/337779 objects degraded (3.172%); 8786/337779 objects misplaced (2.601%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:10:32.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.475 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:32.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:32.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:32 smithi012 ceph-mon[101180]: pgmap v717: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 94 B/s rd, 46 MiB/s wr, 977 op/s; 10713/337779 objects degraded (3.172%); 8786/337779 objects misplaced (2.601%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:10:32.620 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:32.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:32.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:32.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:32.900 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:32.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:33.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:33.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:33.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:33.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:33.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:33.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:33.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:33.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:33.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:33.471 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:33.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:33.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:33.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:33.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:33.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:33.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:10:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:33.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:33.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:33.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:34.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:34.117 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.168 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:34.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:34.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:34.312 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:34.356 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:10:34.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:34.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:34.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[107399]: pgmap v718: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 757 op/s; 10713/334524 objects degraded (3.202%); 8670/334524 objects misplaced (2.592%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2338394974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10713/334524 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[112165]: pgmap v718: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 757 op/s; 10713/334524 objects degraded (3.202%); 8670/334524 objects misplaced (2.592%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2338394974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10713/334524 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:34.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:34.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:34.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:34.694 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:34.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:34.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:34 smithi012 ceph-mon[101180]: pgmap v718: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 757 op/s; 10713/334524 objects degraded (3.202%); 8670/334524 objects misplaced (2.592%); 3.2 MiB/s, 50 objects/s recovering 2024-09-14T04:10:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:34 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2338394974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:34 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:10:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10713/334524 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:34.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:34.946 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:34.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:35.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:35.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:35.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:35.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:35.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:35.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.321 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:35.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:35.368 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:10:35.368 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[107399]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T04:10:35.368 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:35.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:35.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:35.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:35.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:35.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:35.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:10:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[112165]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T04:10:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:35.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:35.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:35 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:10:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:35 smithi012 ceph-mon[101180]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T04:10:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:35.903 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:10:35.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:35.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:35.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:35.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:36.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:36.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:36.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:36.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:36.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:36.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:36.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:36.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:36.452 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:36.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:36 smithi001 ceph-mon[107399]: pgmap v720: 99 pgs: 1 remapped+peering, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 806 op/s; 10713/333051 objects degraded (3.217%); 4428/333051 objects misplaced (1.330%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:36 smithi001 ceph-mon[107399]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T04:10:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:36 smithi001 ceph-mon[112165]: pgmap v720: 99 pgs: 1 remapped+peering, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 806 op/s; 10713/333051 objects degraded (3.217%); 4428/333051 objects misplaced (1.330%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:36 smithi001 ceph-mon[112165]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T04:10:36.476 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:36.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:36.610 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:36.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:36.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:36 smithi012 ceph-mon[101180]: pgmap v720: 99 pgs: 1 remapped+peering, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 806 op/s; 10713/333051 objects degraded (3.217%); 4428/333051 objects misplaced (1.330%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:10:36.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:36 smithi012 ceph-mon[101180]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T04:10:36.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:36.836 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:36] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:10:36.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:36.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:36.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:36.977 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:36.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:37.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:37.020 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:10:37.020 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:10:37.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:37.157 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:37.259 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:37.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:37.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:37.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:37.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:37.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.577 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:37.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:37.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[112165]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T04:10:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[112165]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 208'7516 2024-09-14T04:10:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[112165]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 208'7535 2024-09-14T04:10:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[107399]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T04:10:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[107399]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 208'7516 2024-09-14T04:10:37.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:37 smithi001 ceph-mon[107399]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 208'7535 2024-09-14T04:10:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.736 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:37.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:37.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:37 smithi012 ceph-mon[101180]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T04:10:37.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:37 smithi012 ceph-mon[101180]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 208'7516 2024-09-14T04:10:37.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:37 smithi012 ceph-mon[101180]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 208'7535 2024-09-14T04:10:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:37.896 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:37.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:37.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:38.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:38.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:38.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:38.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:38.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:38.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:38.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:38.574 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[107399]: pgmap v723: 99 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 21 KiB/s wr, 585 op/s; 802/327240 objects degraded (0.245%); 14289/327240 objects misplaced (4.367%); 3.0 MiB/s, 49 objects/s recovering 2024-09-14T04:10:38.574 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[107399]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T04:10:38.574 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:38.574 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[112165]: pgmap v723: 99 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 21 KiB/s wr, 585 op/s; 802/327240 objects degraded (0.245%); 14289/327240 objects misplaced (4.367%); 3.0 MiB/s, 49 objects/s recovering 2024-09-14T04:10:38.574 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[112165]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T04:10:38.575 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:38.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:38.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:38.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:38.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:38 smithi012 ceph-mon[101180]: pgmap v723: 99 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 21 KiB/s wr, 585 op/s; 802/327240 objects degraded (0.245%); 14289/327240 objects misplaced (4.367%); 3.0 MiB/s, 49 objects/s recovering 2024-09-14T04:10:38.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:38 smithi012 ceph-mon[101180]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T04:10:38.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:38.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:38.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:10:38.954 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:38.957 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:38.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:39.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:39.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:39.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:39.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.257 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:39.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:39.397 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:39.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:39.533 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:39 smithi001 ceph-mon[112165]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T04:10:39.533 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 802/327240 objects degraded (0.245%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:39.534 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:39 smithi001 ceph-mon[107399]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T04:10:39.534 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 802/327240 objects degraded (0.245%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:39.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:39.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:39.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:39.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.834 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:39 smithi012 ceph-mon[101180]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T04:10:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 802/327240 objects degraded (0.245%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:39.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:39.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:39.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:40.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:40.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:40.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:40.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:40.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:40.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:40.414 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:10:40.498 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.534 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:40.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:40.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:40.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[107399]: pgmap v726: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 88 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 5094/325431 objects degraded (1.565%); 9861/325431 objects misplaced (3.030%); 2.9 MiB/s, 48 objects/s recovering 2024-09-14T04:10:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3306354593' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[112165]: pgmap v726: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 88 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 5094/325431 objects degraded (1.565%); 9861/325431 objects misplaced (3.030%); 2.9 MiB/s, 48 objects/s recovering 2024-09-14T04:10:40.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3306354593' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.776 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:40.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:40 smithi012 ceph-mon[101180]: pgmap v726: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 88 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 5094/325431 objects degraded (1.565%); 9861/325431 objects misplaced (3.030%); 2.9 MiB/s, 48 objects/s recovering 2024-09-14T04:10:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3306354593' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:10:40.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:40.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:40.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:41.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:41.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:41.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:41.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:41.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:41.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:41.394 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:41.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:41.531 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:41.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:41.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:41.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[107399]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[107399]: pgmap v728: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 5145/318855 objects degraded (1.614%); 31414/318855 objects misplaced (9.852%); 9.5 MiB/s, 151 objects/s recovering 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[112165]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T04:10:41.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:41 smithi001 ceph-mon[112165]: pgmap v728: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 5145/318855 objects degraded (1.614%); 31414/318855 objects misplaced (9.852%); 9.5 MiB/s, 151 objects/s recovering 2024-09-14T04:10:41.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:41.825 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:41 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:10:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:10:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:41 smithi012 ceph-mon[101180]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T04:10:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:41 smithi012 ceph-mon[101180]: pgmap v728: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 5145/318855 objects degraded (1.614%); 31414/318855 objects misplaced (9.852%); 9.5 MiB/s, 151 objects/s recovering 2024-09-14T04:10:41.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:41.969 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:41.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:42.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:42.108 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:42.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:42.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:42.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:42.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:42.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:42.517 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:42.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:42.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:42.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:42.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:42.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:42.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:42.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:43.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:43.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:43.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:43.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:43.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:43.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:43.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:43.485 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:43.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:43.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:43.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:43.722 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:43.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:43.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:43.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:43.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:43.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:43.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:44.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.020 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:44.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:44.191 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:10:44.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:44.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:44 smithi012 ceph-mon[101180]: pgmap v729: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5145/316278 objects degraded (1.627%); 31414/316278 objects misplaced (9.932%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:10:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:44 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:10:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5145/316278 objects degraded (1.627%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:44.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:44.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[112165]: pgmap v729: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5145/316278 objects degraded (1.627%); 31414/316278 objects misplaced (9.932%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5145/316278 objects degraded (1.627%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[107399]: pgmap v729: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5145/316278 objects degraded (1.627%); 31414/316278 objects misplaced (9.932%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:10:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5145/316278 objects degraded (1.627%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:44.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:44.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:44.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:10:44.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:44.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:44.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:44.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:44.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:44.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:44.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:44.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:45.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:45.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:45.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:45.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:45.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:45.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:45.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:10:45.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:45 smithi001 ceph-mon[112165]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T04:10:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:10:45.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:45 smithi001 ceph-mon[107399]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T04:10:45.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:45.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1738327286' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:10:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:45 smithi012 ceph-mon[101180]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T04:10:45.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:45.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:45.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:45.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:45.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:45.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:45.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:46.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:46.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:46.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:46.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:46.389 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:46 smithi001 ceph-mon[107399]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5145/314583 objects degraded (1.635%); 31147/314583 objects misplaced (9.901%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:10:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:46 smithi001 ceph-mon[112165]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5145/314583 objects degraded (1.635%); 31147/314583 objects misplaced (9.901%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:10:46.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:46.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:46 smithi012 ceph-mon[101180]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5145/314583 objects degraded (1.635%); 31147/314583 objects misplaced (9.901%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:10:46.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:46.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:46.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:46.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:46] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:10:46.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:46.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:47.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:47.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:47.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.263 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:47.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:47.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:47.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:47.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:47.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:47.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:47.754 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:47.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:47.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:47.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:48.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:48.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:48.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:48.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:48.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.339 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:48.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:48.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:48 smithi001 ceph-mon[112165]: pgmap v732: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5313/308307 objects degraded (1.723%); 30311/308307 objects misplaced (9.831%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:10:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:48 smithi001 ceph-mon[107399]: pgmap v732: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5313/308307 objects degraded (1.723%); 30311/308307 objects misplaced (9.831%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:10:48.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:48.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:48 smithi012 ceph-mon[101180]: pgmap v732: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5313/308307 objects degraded (1.723%); 30311/308307 objects misplaced (9.831%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:10:48.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:48.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:48.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:48.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:48.882 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:48.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:48.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:49.022 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:49.124 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:49.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:49.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:49.290 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5313/308307 objects degraded (1.723%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:49.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:49.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:10:49.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:49.504 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:49.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:49.508 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5313/308307 objects degraded (1.723%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:49.508 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5313/308307 objects degraded (1.723%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:10:49.534 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:49.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:49.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:49.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:49.798 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:49.900 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:49.960 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:49.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:50.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:50.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:50.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:50.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:50.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:50.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:50.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:50.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[107399]: pgmap v733: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5313/305895 objects degraded (1.737%); 30311/305895 objects misplaced (9.909%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:10:50.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.418 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:50.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[112165]: pgmap v733: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5313/305895 objects degraded (1.737%); 30311/305895 objects misplaced (9.909%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:10:50.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:50.482 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:50.519 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:50.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:50 smithi012 ceph-mon[101180]: pgmap v733: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 5313/305895 objects degraded (1.737%); 30311/305895 objects misplaced (9.909%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:10:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:10:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:50.599 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:10:50.599 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:10:50.621 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:50.640 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:10:50.640 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:10:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:50.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:50.675 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:10:50.676 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:10:50.714 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:10:50.714 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:10:50.747 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:10:50.747 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:10:50.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:50.782 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:10:50.782 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:10:50.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:50.814 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:10:50.815 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:10:50.848 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:10:50.848 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:10:50.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:50.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:50.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:51.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:51.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:51.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:51.213 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:51.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:51.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:51.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:51.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:51.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:51.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:51.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/147937237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:51.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/147937237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/147937237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:10:51.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:51.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:51.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:51.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:51.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:52.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:52.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:52.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:52.180 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:52.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:52.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:52.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:52.430 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:52.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:52.549 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:52 smithi001 ceph-mon[112165]: pgmap v734: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 5515/298584 objects degraded (1.847%); 29129/298584 objects misplaced (9.756%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:52.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:52.550 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:52 smithi001 ceph-mon[107399]: pgmap v734: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 5515/298584 objects degraded (1.847%); 29129/298584 objects misplaced (9.756%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:52.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:52 smithi012 ceph-mon[101180]: pgmap v734: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 5515/298584 objects degraded (1.847%); 29129/298584 objects misplaced (9.756%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:52.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:52.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:52.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:52.886 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:52.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:52.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:53.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:53.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:53.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:53.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.308 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:53.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:53.393 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:10:53.393 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:10:53.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.393 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:133415 10.7 e215/202 10:eddc00d9:::benchmark_data_smithi001_190881_object18275:head v 215'8451, mlcod=0'0)", 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:10:53.384396+0000", 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0066592109999999999, 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0066867630000000001, 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384396+0000", 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384396+0000", 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384398+0000", 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.415e-06 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384405+0000", 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1840000000000002e-06 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384428+0000", 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3538000000000001e-05 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384436+0000", 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6490000000000008e-06 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:10:53.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384491+0000", 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4975000000000001e-05 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384513+0000", 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2025e-05 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:133426 10.5 e215/184 10:ad1fe9cf:::benchmark_data_smithi001_190881_object18286:head v 215'8451, mlcod=215'8446)", 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:10:53.384782+0000", 2024-09-14T04:10:53.398 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0062734799999999997, 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063560429999999996, 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384782+0000", 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384782+0000", 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384783+0000", 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2600000000000002e-07 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384789+0000", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6479999999999997e-06 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384803+0000", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3698e-05 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384810+0000", 2024-09-14T04:10:53.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6429999999999998e-06 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384902+0000", 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1880999999999997e-05 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.384931+0000", 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9176999999999999e-05 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:10:53.402 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:133425 10.7 e215/202 10:e2f8514d:::benchmark_data_smithi001_190881_object18285:head v 215'8452, mlcod=0'0)", 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:10:53.387647+0000", 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0034081960000000001, 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035398019999999999, 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:10:53.403 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:10:53.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:10:53.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387647+0000", 2024-09-14T04:10:53.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387647+0000", 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387648+0000", 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9199999999999999e-07 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387656+0000", 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6690000000000006e-06 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387672+0000", 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5656999999999999e-05 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387680+0000", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2970000000000004e-06 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387890+0000", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021019300000000001 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:10:53.387963+0000", 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2743999999999993e-05 2024-09-14T04:10:53.407 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 3 2024-09-14T04:10:53.408 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:10:53.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:53.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:53.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.582 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:53.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:53.675 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:53.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:53.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:53.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:53.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:53.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:53.905 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:10:53.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:54.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:54.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:54.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:54.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:54.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:54.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:54 smithi001 ceph-mon[107399]: pgmap v735: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5515/295941 objects degraded (1.864%); 29129/295941 objects misplaced (9.843%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:54.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5515/295941 objects degraded (1.864%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:54.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:54 smithi001 ceph-mon[112165]: pgmap v735: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5515/295941 objects degraded (1.864%); 29129/295941 objects misplaced (9.843%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:54.397 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5515/295941 objects degraded (1.864%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:54.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:10:54.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:54.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:54 smithi012 ceph-mon[101180]: pgmap v735: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5515/295941 objects degraded (1.864%); 29129/295941 objects misplaced (9.843%); 6.6 MiB/s, 106 objects/s recovering 2024-09-14T04:10:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5515/295941 objects degraded (1.864%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:54.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:54.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.712 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:54.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:54.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:54.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:54.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:54.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:54.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:54.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:55.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:55.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:55.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:55.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:55.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:55.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:55.405 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:55.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:55.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:55.649 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:55.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:55.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:55.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:55.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:55.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:55.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:55.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:56.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:56.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:56.208 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:56.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:56.346 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:56.382 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:56.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:56.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:56 smithi012 ceph-mon[101180]: pgmap v736: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5515/294024 objects degraded (1.876%); 28823/294024 objects misplaced (9.803%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:10:56.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:56.708 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:56 smithi001 ceph-mon[112165]: pgmap v736: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5515/294024 objects degraded (1.876%); 28823/294024 objects misplaced (9.803%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:10:56.708 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:56 smithi001 ceph-mon[107399]: pgmap v736: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5515/294024 objects degraded (1.876%); 28823/294024 objects misplaced (9.803%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:10:56.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:56.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:56.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:56.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:10:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:10:56] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-09-14T04:10:57.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:57.080 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:57.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:57.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:57.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:57.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:57.409 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:57.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:57.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:57.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:57.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:57.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:57.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:57.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:57.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:57.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:58.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:58.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:58.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:58.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:58.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.245 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:58.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:58.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:10:58.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:10:58.470 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:10:58.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:58.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:58.562 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:58 smithi001 ceph-mon[107399]: pgmap v737: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5724/287172 objects degraded (1.993%); 27912/287172 objects misplaced (9.720%); 7.1 MiB/s, 114 objects/s recovering 2024-09-14T04:10:58.562 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:58 smithi001 ceph-mon[112165]: pgmap v737: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5724/287172 objects degraded (1.993%); 27912/287172 objects misplaced (9.720%); 7.1 MiB/s, 114 objects/s recovering 2024-09-14T04:10:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:58 smithi012 ceph-mon[101180]: pgmap v737: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5724/287172 objects degraded (1.993%); 27912/287172 objects misplaced (9.720%); 7.1 MiB/s, 114 objects/s recovering 2024-09-14T04:10:58.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:58.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:10:58.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:10:58.849 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:58.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:10:58.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:58.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:58.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:59.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:10:59.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:10:59.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:10:59.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:10:59.282 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:10:59.296 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:10:59.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:59.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:59.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:10:59.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:59.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:10:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:10:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5724/287172 objects degraded (1.993%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:10:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:10:59.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:10:59.680 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:59.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5724/287172 objects degraded (1.993%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5724/287172 objects degraded (1.993%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:10:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:10:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:10:59.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:10:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:10:59.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:10:59.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:10:59.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:10:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:10:59.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:00.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:00.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:00.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:00.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:00.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:00.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:00.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:00 smithi012 ceph-mon[101180]: pgmap v738: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5724/284433 objects degraded (2.012%); 27912/284433 objects misplaced (9.813%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:11:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:00 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:11:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:00 smithi012 ceph-mon[101180]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T04:11:00.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:00.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[112165]: pgmap v738: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5724/284433 objects degraded (2.012%); 27912/284433 objects misplaced (9.813%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:11:00.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:00.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:11:00.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:00.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[112165]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T04:11:00.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[107399]: pgmap v738: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5724/284433 objects degraded (2.012%); 27912/284433 objects misplaced (9.813%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:11:00.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:00.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2986902731' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:11:00.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2974227189' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:00.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:00 smithi001 ceph-mon[107399]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T04:11:00.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:00.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:00.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:00.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:00.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:00.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:00.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:01.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:01.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:01.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:01.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:01.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:01.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:01.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:01.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:01.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:01.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:01.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:01.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:01.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:01.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:01.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:01.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:02.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:02.058 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:02.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:02.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:02.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:02.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:02.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:02.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:02.586 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:02 smithi012 ceph-mon[101180]: pgmap v740: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 5891/276933 objects degraded (2.127%); 26702/276933 objects misplaced (9.642%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:02.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:02.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:02 smithi001 ceph-mon[107399]: pgmap v740: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 5891/276933 objects degraded (2.127%); 26702/276933 objects misplaced (9.642%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:02 smithi001 ceph-mon[112165]: pgmap v740: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 5891/276933 objects degraded (2.127%); 26702/276933 objects misplaced (9.642%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:02.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.782 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:02.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:02.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:11:02.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:02.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:02.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:03.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:03.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:03.123 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:11:03.147 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:11:03.147 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:11:03.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:03.187 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:11:03.187 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:11:03.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:03.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:03.236 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:11:03.236 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:11:03.272 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:11:03.272 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:11:03.308 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:11:03.308 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:11:03.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:03.344 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:11:03.344 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:11:03.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:03.383 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:11:03.383 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:11:03.419 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2217616569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:03.419 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2217616569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:03.422 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:11:03.422 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:11:03.472 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:03.529 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:03.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2217616569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:03.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:03.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:03.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:03.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:03.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:03.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:03.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:03.984 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:11:03.984 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:11:04.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:04.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:04.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:04.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:04.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:04.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:04.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:04.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:04 smithi012 ceph-mon[101180]: pgmap v741: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 5891/274008 objects degraded (2.150%); 26702/274008 objects misplaced (9.745%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5891/274008 objects degraded (2.150%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:04.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:04.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:04 smithi001 ceph-mon[112165]: pgmap v741: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 5891/274008 objects degraded (2.150%); 26702/274008 objects misplaced (9.745%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5891/274008 objects degraded (2.150%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:04 smithi001 ceph-mon[107399]: pgmap v741: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 5891/274008 objects degraded (2.150%); 26702/274008 objects misplaced (9.745%); 7.5 MiB/s, 119 objects/s recovering 2024-09-14T04:11:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5891/274008 objects degraded (2.150%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:04.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:04.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:04.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:04.934 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:04.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:05.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:05.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:05.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:05.213 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:05.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:05.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:05.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:05.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:05.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:05.632 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:05.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:05.771 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:05.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:05.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:05.928 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:05.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:05.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:11:06.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:06.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:06.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:06.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:06.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[107399]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5891/272055 objects degraded (2.165%); 26347/272055 objects misplaced (9.684%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:05 smithi001 ceph-mon[112165]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5891/272055 objects degraded (2.165%); 26347/272055 objects misplaced (9.684%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:06.277 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:06.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:05 smithi012 ceph-mon[101180]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5891/272055 objects degraded (2.165%); 26347/272055 objects misplaced (9.684%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:06.367 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:06.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:06.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:06.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:11:06.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:06.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:06.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:06.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:06.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:06.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:06.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:06] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.51.0" 2024-09-14T04:11:06.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:06.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:07.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:07.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:07.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:07.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:07.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:07.408 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:07.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:07.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:07.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:07.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:07.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:07.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:07.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:07.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:07.958 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:07.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:08.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.094 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:08.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:08.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:08.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:08.336 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:08.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:08 smithi001 ceph-mon[107399]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 6092/265155 objects degraded (2.298%); 25434/265155 objects misplaced (9.592%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:08 smithi001 ceph-mon[112165]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 6092/265155 objects degraded (2.298%); 25434/265155 objects misplaced (9.592%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:08.473 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:08.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:08 smithi012 ceph-mon[101180]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 6092/265155 objects degraded (2.298%); 25434/265155 objects misplaced (9.592%); 7.8 MiB/s, 127 objects/s recovering 2024-09-14T04:11:08.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:08.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:08.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:08.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:08.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:08.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:08.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:08.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:09.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:09.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:09.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:09.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:09.211 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:09.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:09.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:09.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:09.442 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:09.465 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:09.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6092/265155 objects degraded (2.298%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:09.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:09 smithi001 ceph-mon[107399]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T04:11:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6092/265155 objects degraded (2.298%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:09 smithi001 ceph-mon[112165]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T04:11:09.567 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:09.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:09.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6092/265155 objects degraded (2.298%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:09 smithi012 ceph-mon[101180]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T04:11:09.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:09.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:09.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:09.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:09.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:09.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:09.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:10.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.118 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:10.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:10.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:10.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:10.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:10.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:10 smithi001 ceph-mon[107399]: pgmap v745: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6092/262509 objects degraded (2.321%); 25434/262509 objects misplaced (9.689%); 7.9 MiB/s, 129 objects/s recovering 2024-09-14T04:11:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:10 smithi001 ceph-mon[107399]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T04:11:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:10 smithi001 ceph-mon[112165]: pgmap v745: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6092/262509 objects degraded (2.321%); 25434/262509 objects misplaced (9.689%); 7.9 MiB/s, 129 objects/s recovering 2024-09-14T04:11:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:10 smithi001 ceph-mon[112165]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T04:11:10.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.564 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:10.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:10 smithi012 ceph-mon[101180]: pgmap v745: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 6092/262509 objects degraded (2.321%); 25434/262509 objects misplaced (9.689%); 7.9 MiB/s, 129 objects/s recovering 2024-09-14T04:11:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:10 smithi012 ceph-mon[101180]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T04:11:10.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:10.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:10.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:10.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:10.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:10.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:10.968 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:10.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:10.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:11.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:11.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:11.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:11.155 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:11.229 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:11:11.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:11.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:11.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:11.395 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:11.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:11.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:11.513 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:11.513 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:11.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:11.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:11.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:11.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.841 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:11.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:11.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:11.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:11.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:12.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:12.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:12.239 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:12.245 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:11:12.278 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:12.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:12.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 6230/256263 objects degraded (2.431%); 24589/256263 objects misplaced (9.595%); 7.4 MiB/s, 168 objects/s recovering 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3845771073' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 6230/256263 objects degraded (2.431%); 24589/256263 objects misplaced (9.595%); 7.4 MiB/s, 168 objects/s recovering 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3845771073' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:12.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 6230/256263 objects degraded (2.431%); 24589/256263 objects misplaced (9.595%); 7.4 MiB/s, 168 objects/s recovering 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1155703559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3845771073' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:11:12.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:12.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:12.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:12.798 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:12.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:12.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:12.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:13.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:13.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:13.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:13.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:13.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:13.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:13.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:13.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:13 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:11:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:13 smithi001 ceph-mon[107399]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T04:11:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:13 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:11:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:13 smithi001 ceph-mon[112165]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T04:11:13.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:13.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:13 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:11:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:13 smithi012 ceph-mon[101180]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T04:11:13.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:13.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:13.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:13.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:13.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:13.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:13.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:14.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:14.117 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:14.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:14.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:14.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:14.393 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:14.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:14.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:14 smithi012 ceph-mon[101180]: pgmap v750: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 6230/254025 objects degraded (2.453%); 24589/254025 objects misplaced (9.680%); 4.8 MiB/s, 165 objects/s recovering 2024-09-14T04:11:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:14 smithi012 ceph-mon[101180]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T04:11:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6230/254025 objects degraded (2.453%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:14.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:14.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[107399]: pgmap v750: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 6230/254025 objects degraded (2.453%); 24589/254025 objects misplaced (9.680%); 4.8 MiB/s, 165 objects/s recovering 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[107399]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6230/254025 objects degraded (2.453%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[112165]: pgmap v750: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 6230/254025 objects degraded (2.453%); 24589/254025 objects misplaced (9.680%); 4.8 MiB/s, 165 objects/s recovering 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[112165]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T04:11:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6230/254025 objects degraded (2.453%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:14.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:14.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:14.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:14.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:14.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:15.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:15.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:15.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:15.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:15.239 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:11:15.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:15.488 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:15.531 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:15 smithi012 ceph-mon[101180]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T04:11:15.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:15.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:15 smithi001 ceph-mon[107399]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T04:11:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:15 smithi001 ceph-mon[112165]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T04:11:15.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:15.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:15.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:15.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:15.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:16.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:16.082 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:16.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:16.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:16.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:16.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:16.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:16.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:16 smithi012 ceph-mon[101180]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 6230/252327 objects degraded (2.469%); 24589/252327 objects misplaced (9.745%) 2024-09-14T04:11:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:16 smithi012 ceph-mon[101180]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T04:11:16.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:16.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:16.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:16 smithi001 ceph-mon[107399]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 6230/252327 objects degraded (2.469%); 24589/252327 objects misplaced (9.745%) 2024-09-14T04:11:16.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:16 smithi001 ceph-mon[107399]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T04:11:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:16 smithi001 ceph-mon[112165]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 6230/252327 objects degraded (2.469%); 24589/252327 objects misplaced (9.745%) 2024-09-14T04:11:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:16 smithi001 ceph-mon[112165]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T04:11:16.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:16.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:16.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:16.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:16.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:16] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.51.0" 2024-09-14T04:11:17.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:17.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:17.153 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:17.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:17.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:17.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:17.326 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:17.428 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:17.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:17.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:17.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:17 smithi012 ceph-mon[101180]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T04:11:17.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:17.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:17 smithi001 ceph-mon[107399]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T04:11:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:17 smithi001 ceph-mon[112165]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T04:11:17.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:17.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:17.890 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:17.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:17.991 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:18.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:18.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:18.160 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:18.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:18.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:11:18.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:18.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:18.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:18.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:18.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:18 smithi001 ceph-mon[107399]: pgmap v756: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9175/246228 objects degraded (3.726%); 20671/246228 objects misplaced (8.395%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T04:11:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:18 smithi001 ceph-mon[112165]: pgmap v756: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9175/246228 objects degraded (3.726%); 20671/246228 objects misplaced (8.395%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T04:11:18.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:18.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:18 smithi012 ceph-mon[101180]: pgmap v756: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9175/246228 objects degraded (3.726%); 20671/246228 objects misplaced (8.395%); 3.6 MiB/s, 57 objects/s recovering 2024-09-14T04:11:18.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:18.746 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:18.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:18.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:18.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:18.967 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:18.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:19.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:19.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:19.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:19.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:19.285 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:19.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:19.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:19.439 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9175/246228 objects degraded (3.726%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:19.440 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9175/246228 objects degraded (3.726%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:19.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:19.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:19.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9175/246228 objects degraded (3.726%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:19.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:19.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:19.746 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:19.848 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:19.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:19.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:19.991 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:11:19.991 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:11:20.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:20.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:20.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:20.144 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:20.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:20.291 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:20.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:20.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:20.463 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:20.463 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:20.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:20.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:20.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:20.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: pgmap v757: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 9175/243507 objects degraded (3.768%); 20671/243507 objects misplaced (8.489%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: pgmap v757: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 9175/243507 objects degraded (3.768%); 20671/243507 objects misplaced (8.489%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:20.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:20.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: pgmap v757: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 9175/243507 objects degraded (3.768%); 20671/243507 objects misplaced (8.489%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:20.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:20.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:20.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:20.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:21.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:21.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:11:21.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:21.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:21.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:21.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:21.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:21.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:21.456 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:21.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:21.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:21.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:21.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:21.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:21.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:21.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:21.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:21.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:22.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:22.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:22.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:22.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.302 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:22.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:22.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.475 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:22.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:22.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:22.607 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:22 smithi001 ceph-mon[107399]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 9291/235296 objects degraded (3.949%); 19906/235296 objects misplaced (8.460%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:11:22.607 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:22 smithi001 ceph-mon[112165]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 9291/235296 objects degraded (3.949%); 19906/235296 objects misplaced (8.460%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:11:22.615 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:22.717 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:22.756 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:22 smithi012 ceph-mon[101180]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 9291/235296 objects degraded (3.949%); 19906/235296 objects misplaced (8.460%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:11:22.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:22.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:22.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:22.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:23.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:23.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.180 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:23.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:23.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:23.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:23.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:23.461 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:23.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:23.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:23.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:23.722 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:11:23.722 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:11:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:23.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:23.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:23.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:23.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:23.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:24.021 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:24.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:24.155 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:24.162 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:11:24.162 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:11:24.199 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:11:24.199 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:11:24.238 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:11:24.239 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:11:24.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:24.271 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:11:24.272 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:11:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:24.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:24.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:11:24.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:11:24.341 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:11:24.341 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:11:24.376 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:11:24.376 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:11:24.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:24.407 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:11:24.408 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:11:24.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:24.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:11:24.500 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[107399]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 9291/232356 objects degraded (3.999%); 19906/232356 objects misplaced (8.567%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:11:24.500 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9291/232356 objects degraded (3.999%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:24.500 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4080705796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:24.501 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[112165]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 9291/232356 objects degraded (3.999%); 19906/232356 objects misplaced (8.567%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:11:24.501 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9291/232356 objects degraded (3.999%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:24.501 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4080705796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:24.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:24.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:24.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.763 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:24.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:24 smithi012 ceph-mon[101180]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 9291/232356 objects degraded (3.999%); 19906/232356 objects misplaced (8.567%); 2.9 MiB/s, 47 objects/s recovering 2024-09-14T04:11:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9291/232356 objects degraded (3.999%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4080705796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:24.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:24.885 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:24.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:24.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:25.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:25.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:25.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:25.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:25.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.183 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:25.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:25.285 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:25.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:25.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:25.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:25.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:25.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:25.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:25.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:25.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.739 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:25.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:25.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:25.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:25.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:25.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:26.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:26.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:26.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:26.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:26.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:26.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:26.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:26.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:26.583 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:26.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:26 smithi001 ceph-mon[107399]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 9291/230112 objects degraded (4.038%); 19906/230112 objects misplaced (8.651%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:26.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:26 smithi001 ceph-mon[112165]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 9291/230112 objects degraded (4.038%); 19906/230112 objects misplaced (8.651%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:26 smithi012 ceph-mon[101180]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 9291/230112 objects degraded (4.038%); 19906/230112 objects misplaced (8.651%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:26.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:26.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:26.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:26.853 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:26] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-09-14T04:11:26.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:26.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:26.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:26.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.015 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:27.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:27.093 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:11:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:11:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:11:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:11:27.093 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:11:27.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:27.159 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:27.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:27.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:27.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:27.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:27.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:27.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.591 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:27.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:27.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:27.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:27.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:11:27.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:27.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:27.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:27.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:28.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:28.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:11:28.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:28.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:28.157 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:28.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:28.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:28.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:28.415 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:11:28.417 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:11:28.425 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:28.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:28.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:28.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:28.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[107399]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9392/222753 objects degraded (4.216%); 19201/222753 objects misplaced (8.620%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:11:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:11:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:11:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[112165]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9392/222753 objects degraded (4.216%); 19201/222753 objects misplaced (8.620%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:11:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:11:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:11:28.733 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:28.742 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:28 smithi012 ceph-mon[101180]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9392/222753 objects degraded (4.216%); 19201/222753 objects misplaced (8.620%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:11:28.742 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:11:28.742 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:11:28.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:28.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:28.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:28.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:28.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:29.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:29.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.148 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:29.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:29.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:29.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:29.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:29.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:29.430 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:29.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:29.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:29.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:29.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:29.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9392/222753 objects degraded (4.216%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[112165]: pgmap v763: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 9392/219780 objects degraded (4.273%); 19201/219780 objects misplaced (8.736%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:11:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:11:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T04:11:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9392/222753 objects degraded (4.216%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:29 smithi001 ceph-mon[107399]: pgmap v763: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 9392/219780 objects degraded (4.273%); 19201/219780 objects misplaced (8.736%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:29.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:29.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:29.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2963442123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/885761708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9392/222753 objects degraded (4.216%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:29 smithi012 ceph-mon[101180]: pgmap v763: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 9392/219780 objects degraded (4.273%); 19201/219780 objects misplaced (8.736%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:29.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:29.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:30.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:30.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.146 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:30.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:30.247 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:30.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:30.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:11:30.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:30.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:30.530 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:30.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:163257 9.c 9:33771beb:::benchmark_data_smithi001_156540_object31663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:25.641651+0000", 2024-09-14T04:11:30.653 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.00565312499998, 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.106820341, 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 163257 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641651+0000", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641651+0000", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641649+0000", 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641656+0000", 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1450000000000002e-06 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641676+0000", 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9930000000000001e-05 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641684+0000", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6450000000000005e-06 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641823+0000", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013949599999999999 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.641966+0000", 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014284200000000001 2024-09-14T04:11:30.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.642091+0000", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012525700000000001 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.647558+0000", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054668929999999996 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.647916+0000", 2024-09-14T04:11:30.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035818200000000002 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.747614+0000", 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099697456000000004 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.747694+0000", 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0123000000000005e-05 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.748471+0000", 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00077697200000000001 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166808 9.1 e153/149 9:85c01d48:::benchmark_data_smithi001_156540_object35214:head v 153'10485, mlcod=153'10481)", 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.231292+0000", 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 300.41601140400002, 2024-09-14T04:11:30.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15030952, 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231292+0000", 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231292+0000", 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231291+0000", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231301+0000", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2890000000000003e-06 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231311+0000", 2024-09-14T04:11:30.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0118e-05 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231317+0000", 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6529999999999997e-06 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231471+0000", 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153714 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.231493+0000", 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1461999999999999e-05 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381513+0000", 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.150020551 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381602+0000", 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8802999999999997e-05 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.664 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178533 9.c 9.88075e2c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.342744+0000", 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 285.30455940399997, 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98660115100000001, 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:30.665 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178533 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342744+0000", 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342744+0000", 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342743+0000", 2024-09-14T04:11:30.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342750+0000", 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.934e-06 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342762+0000", 2024-09-14T04:11:30.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.203e-05 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.342769+0000", 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3900000000000004e-06 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.344053+0000", 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012843259999999999 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.344066+0000", 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2332e-05 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.329332+0000", 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98526634800000001 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.329345+0000", 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3375e-05 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:220059 9.c 9:39324b8d:::benchmark_data_smithi001_156540_object88465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:07:30.010445+0000", 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 240.636858628, 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12704495499999999, 2024-09-14T04:11:30.671 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 220059 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010445+0000", 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010445+0000", 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010443+0000", 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010453+0000", 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0312000000000001e-05 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010476+0000", 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2249e-05 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010486+0000", 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0536999999999999e-05 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010537+0000", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1338000000000002e-05 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010702+0000", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000165026 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010841+0000", 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013845400000000001 2024-09-14T04:11:30.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.017028+0000", 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0061866999999999998 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.019439+0000", 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024114420000000002 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.137406+0000", 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.117967479 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.137455+0000", 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8174999999999999e-05 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.137490+0000", 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5416000000000001e-05 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21879 10.4 10:2c511003:::benchmark_data_smithi001_190881_object21878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.69347217500001, 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98974571700000002, 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.678 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21879 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953830+0000", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953860+0000", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0428000000000001e-05 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953872+0000", 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2091e-05 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953878+0000", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1179999999999997e-06 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953952+0000", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3336000000000004e-05 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953965+0000", 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3721e-05 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940022+0000", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.986057038 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940143+0000", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121018 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940236+0000", 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2322000000000001e-05 2024-09-14T04:11:30.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943389+0000", 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003153061 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943469+0000", 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0409999999999998e-05 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943481+0000", 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1846e-05 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943537+0000", 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6239000000000001e-05 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943577+0000", 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.002e-05 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21888 10.4 10:2d9d53d0:::benchmark_data_smithi001_190881_object21887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.68823950399999, 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.984828967, 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21888 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959063+0000", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959103+0000", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0454999999999998e-05 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959118+0000", 2024-09-14T04:11:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4647999999999999e-05 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959126+0000", 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7489999999999998e-06 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959247+0000", 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121421 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959274+0000", 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6741e-05 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941501+0000", 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98222701700000004 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941620+0000", 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011943999999999999 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941777+0000", 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015683900000000001 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943700+0000", 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019231770000000001 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943739+0000", 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8143999999999999e-05 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943805+0000", 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6345000000000003e-05 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943856+0000", 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0800000000000002e-05 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943893+0000", 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7509999999999998e-05 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21892 10.4 10:223580ea:::benchmark_data_smithi001_190881_object21891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.68622197400001, 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.983866295, 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.694 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21892 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961081+0000", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961113+0000", 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2369999999999997e-05 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961127+0000", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3575e-05 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961134+0000", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.029e-06 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961201+0000", 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7700999999999994e-05 2024-09-14T04:11:30.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961214+0000", 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2417e-05 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942369+0000", 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98115569800000002 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942480+0000", 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011027900000000001 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942622+0000", 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014186799999999999 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943796+0000", 2024-09-14T04:11:30.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011746479999999999 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943905+0000", 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109042 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944865+0000", 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00096004200000000004 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944914+0000", 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8717999999999997e-05 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944948+0000", 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4029999999999998e-05 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.701 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21898 10.4 10:2228a969:::benchmark_data_smithi001_190881_object21897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.68263904599999, 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98440279799999997, 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21898 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964663+0000", 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964713+0000", 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0379e-05 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964726+0000", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.296e-05 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964734+0000", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2109999999999998e-06 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964799+0000", 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4608000000000002e-05 2024-09-14T04:11:30.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964811+0000", 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1708999999999999e-05 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943932+0000", 2024-09-14T04:11:30.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97912149999999998 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944038+0000", 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010564 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944169+0000", 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013061 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946888+0000", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027196109999999998 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946920+0000", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1519000000000001e-05 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948979+0000", 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020592879999999998 2024-09-14T04:11:30.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949026+0000", 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7139999999999999e-05 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949067+0000", 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1396000000000001e-05 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21908 10.4 10:2c8e17a2:::benchmark_data_smithi001_190881_object21907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.67722455200001, 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97906692100000003, 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21908 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970077+0000", 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970145+0000", 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7878000000000003e-05 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970162+0000", 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6841999999999999e-05 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970169+0000", 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8739999999999997e-06 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970225+0000", 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5591999999999999e-05 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970236+0000", 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1036000000000001e-05 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944976+0000", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97474024699999995 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945077+0000", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101099 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945177+0000", 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9667999999999995e-05 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946948+0000", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00177119 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946976+0000", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7722e-05 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949084+0000", 2024-09-14T04:11:30.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021082760000000001 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949114+0000", 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9964999999999999e-05 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949146+0000", 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2264999999999997e-05 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.716 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21912 10.4 10:259478c7:::benchmark_data_smithi001_190881_object21911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.67652017899999, 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97845593200000003, 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21912 2024-09-14T04:11:30.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.718 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970782+0000", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970811+0000", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8303999999999999e-05 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970825+0000", 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4528e-05 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970833+0000", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0420000000000003e-06 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970934+0000", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100554 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970947+0000", 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3183000000000001e-05 2024-09-14T04:11:30.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945999+0000", 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.975052319 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946112+0000", 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112748 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946241+0000", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129011 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948925+0000", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026834599999999999 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948969+0000", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4746999999999997e-05 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949156+0000", 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018685199999999999 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949203+0000", 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6959999999999998e-05 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949239+0000", 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6415999999999998e-05 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.723 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21955 10.4 10:23191742:::benchmark_data_smithi001_190881_object21954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.65459865, 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95687778000000001, 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21955 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.724 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992704+0000", 2024-09-14T04:11:30.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992741+0000", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7347e-05 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992754+0000", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3004e-05 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992762+0000", 2024-09-14T04:11:30.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9799999999999998e-06 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992815+0000", 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3074e-05 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992828+0000", 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2693e-05 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946995+0000", 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95416745300000005 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947103+0000", 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010747 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947219+0000", 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116579 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949249+0000", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002029515 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949303+0000", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4493999999999997e-05 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949394+0000", 2024-09-14T04:11:30.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0954999999999999e-05 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949450+0000", 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5770999999999998e-05 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949583+0000", 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013270900000000001 2024-09-14T04:11:30.731 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22005 10.4 10:249d7abb:::benchmark_data_smithi001_190881_object22004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 142.61060776900001, 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91457057200000003, 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.732 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22005 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036695+0000", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036741+0000", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6712999999999998e-05 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036753+0000", 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1610000000000001e-05 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036758+0000", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3249999999999998e-06 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036799+0000", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0995000000000003e-05 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036807+0000", 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1119999999999996e-06 2024-09-14T04:11:30.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947971+0000", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.911163905 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948103+0000", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131345 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948253+0000", 2024-09-14T04:11:30.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014980700000000001 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.950100+0000", 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018479449999999999 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.950128+0000", 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7174000000000001e-05 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951184+0000", 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010561119999999999 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951228+0000", 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4464e-05 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951267+0000", 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8315000000000002e-05 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.738 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22411 10.4 10:281e4e94:::benchmark_data_smithi001_190881_object22410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:09.969631+0000", 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 140.677672511, 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.109774131, 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22411 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969631+0000", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969631+0000", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969630+0000", 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969668+0000", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8489999999999999e-05 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969694+0000", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.548e-05 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969702+0000", 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2579999999999995e-06 2024-09-14T04:11:30.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969776+0000", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4220000000000004e-05 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.969942+0000", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016576300000000001 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.970093+0000", 2024-09-14T04:11:30.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150796 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.971409+0000", 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013161679999999999 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.971698+0000", 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028959299999999999 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:10.079306+0000", 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107608116 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:10.079364+0000", 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7921999999999999e-05 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:10.079405+0000", 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0978e-05 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.744 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:45567 10.4 10:2d17cc11:::benchmark_data_smithi001_190881_object45566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:21.448993+0000", 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 129.19831040099999, 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.105725597, 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 45567 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.745 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.448993+0000", 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.448993+0000", 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.448992+0000", 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449028+0000", 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5927999999999999e-05 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449046+0000", 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8289000000000001e-05 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449055+0000", 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0240000000000003e-06 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449098+0000", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2357999999999997e-05 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449288+0000", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019081400000000001 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.449426+0000", 2024-09-14T04:11:30.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013765000000000001 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.451685+0000", 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022590700000000002 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.451917+0000", 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023240199999999999 2024-09-14T04:11:30.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.554605+0000", 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102687557 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.554667+0000", 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1690000000000001e-05 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:21.554719+0000", 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2191000000000002e-05 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:112635 10.4 10:281d0f22:::benchmark_data_smithi001_190881_object112634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:23.346358+0000", 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 67.300945244000005, 2024-09-14T04:11:30.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.093800986000000003, 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 112635 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346358+0000", 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346358+0000", 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346357+0000", 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346394+0000", 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7410999999999998e-05 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346410+0000", 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5162e-05 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346417+0000", 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9890000000000003e-06 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346467+0000", 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0705000000000003e-05 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346609+0000", 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141452 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.346759+0000", 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150051 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.350245+0000", 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034859750000000001 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.351705+0000", 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001460666 2024-09-14T04:11:30.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440085+0000", 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.088379831000000006 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440125+0000", 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9342999999999999e-05 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440159+0000", 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4894000000000002e-05 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:112648 10.4 10:20f5922e:::benchmark_data_smithi001_190881_object112647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:23.361159+0000", 2024-09-14T04:11:30.760 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 67.286145082000004, 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.079109674000000005, 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 112648 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361159+0000", 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361159+0000", 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361157+0000", 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361219+0000", 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2327000000000001e-05 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361239+0000", 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9689000000000001e-05 2024-09-14T04:11:30.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361248+0000", 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1050000000000001e-06 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361307+0000", 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9378e-05 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361468+0000", 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016059099999999999 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.361626+0000", 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015806500000000001 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.365809+0000", 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041832939999999997 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.366422+0000", 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00061266399999999996 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440223+0000", 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073800836999999994 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440247+0000", 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4476000000000001e-05 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:23.440268+0000", 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1271e-05 2024-09-14T04:11:30.767 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:113522 10.e 10:71aaf26f:::benchmark_data_smithi001_190881_object113521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:24.592375+0000", 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 66.054928896000007, 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.085239876000000006, 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.768 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 113522 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592375+0000", 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592375+0000", 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592373+0000", 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592412+0000", 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9007000000000001e-05 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592426+0000", 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4452e-05 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.592435+0000", 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3299999999999999e-06 2024-09-14T04:11:30.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.597874+0000", 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054397279999999996 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598384+0000", 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051006000000000001 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.599010+0000", 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062597599999999998 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.607843+0000", 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0088325880000000006 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.608157+0000", 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031354400000000002 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.677558+0000", 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069401768000000003 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.677590+0000", 2024-09-14T04:11:30.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1983999999999997e-05 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.677615+0000", 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4383999999999999e-05 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:113528 10.e 10:7558a95a:::benchmark_data_smithi001_190881_object113527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.775 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:24.598223+0000", 2024-09-14T04:11:30.776 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 66.049081232000006, 2024-09-14T04:11:30.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.096415756000000005, 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.777 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 113528 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598223+0000", 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598223+0000", 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598222+0000", 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598250+0000", 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8470999999999999e-05 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598270+0000", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9661e-05 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.598277+0000", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.686e-06 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.603528+0000", 2024-09-14T04:11:30.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052511889999999999 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.603716+0000", 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000188419 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.603909+0000", 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019261899999999999 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.608150+0000", 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042410570000000003 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.608161+0000", 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1324e-05 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.694550+0000", 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.086388488999999999 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.694601+0000", 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1066000000000002e-05 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.694638+0000", 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7574999999999999e-05 2024-09-14T04:11:30.783 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:113785 10.e 10:7c3e3db7:::benchmark_data_smithi001_190881_object113784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:24.974946+0000", 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 65.672358013999997, 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073065929000000002, 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.784 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 113785 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974946+0000", 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974946+0000", 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974944+0000", 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974975+0000", 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0490000000000001e-05 2024-09-14T04:11:30.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974991+0000", 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.645e-05 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.974999+0000", 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.975053+0000", 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4268999999999998e-05 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.975212+0000", 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015918400000000001 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.975336+0000", 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000123468 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.987157+0000", 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011821757 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.987279+0000", 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121923 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.047919+0000", 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060639480000000003 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.047972+0000", 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2936999999999998e-05 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.048012+0000", 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0014e-05 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:113789 10.e 10:711be826:::benchmark_data_smithi001_190881_object113788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:10:24.977183+0000", 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 65.670120627000003, 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070964071000000004, 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 113789 2024-09-14T04:11:30.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977183+0000", 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977183+0000", 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977182+0000", 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977242+0000", 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9697999999999997e-05 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:30.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977252+0000", 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0203999999999999e-05 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.977259+0000", 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0049999999999998e-06 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.979268+0000", 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020097330000000001 2024-09-14T04:11:30.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.979419+0000", 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150056 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.979557+0000", 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013872600000000001 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.989993+0000", 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010435381000000001 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:24.991254+0000", 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001260866 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:30.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.048058+0000", 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056804070999999998 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.048110+0000", 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2232999999999999e-05 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:10:25.048147+0000", 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7370999999999998e-05 2024-09-14T04:11:30.798 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.799 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:30.799 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:30.799 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:11:30.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:30.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:30.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:30.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:30.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:31.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:31.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:31.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.216 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:31.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:31.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:31.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:31.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:31.487 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:31.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:31.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:31.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:31.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:31.766 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:31.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:31.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:31.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:32.010 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:32.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:32.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:32.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:32.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:32.279 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:11:32.280 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:11:32.288 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:11:32.289 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:11:32.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:32.319 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:11:32.320 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:11:32.320 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:32 smithi001 ceph-mon[112165]: pgmap v764: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9489/211839 objects degraded (4.479%); 18477/211839 objects misplaced (8.722%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:32.320 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:32 smithi001 ceph-mon[107399]: pgmap v764: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9489/211839 objects degraded (4.479%); 18477/211839 objects misplaced (8.722%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:32.354 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:11:32.354 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:11:32.376 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:11:32.377 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:11:32.397 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:11:32.398 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:11:32.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:32.444 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:11:32.445 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:11:32.460 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:32.482 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:11:32.483 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:11:32.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:32 smithi012 ceph-mon[101180]: pgmap v764: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9489/211839 objects degraded (4.479%); 18477/211839 objects misplaced (8.722%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:32.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:32.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:32.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:32.740 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:32.842 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:32.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:32.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:32.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:11:32.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:33.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:33.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:33.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:33.209 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:33 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/643073379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:33.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:33.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:33.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:33 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/643073379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:33.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:33.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:33.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:33 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/643073379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:33.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.582 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:33.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:33.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:33.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:33.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:33.829 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:33.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:33.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:33.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:11:33.966 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:34.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:34.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:34.136 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:34.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:34.270 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:34.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.271 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:34.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:34 smithi001 ceph-mon[107399]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9489/208941 objects degraded (4.541%); 18477/208941 objects misplaced (8.843%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:34.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9489/208941 objects degraded (4.541%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:34.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:34 smithi001 ceph-mon[112165]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9489/208941 objects degraded (4.541%); 18477/208941 objects misplaced (8.843%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:34.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9489/208941 objects degraded (4.541%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:34.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:34.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:34.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:34.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:34.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:34 smithi012 ceph-mon[101180]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9489/208941 objects degraded (4.541%); 18477/208941 objects misplaced (8.843%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:34.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9489/208941 objects degraded (4.541%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:34.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:34.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:34.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:34.850 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:34.869 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:34.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:34.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:34.992 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:34.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:35.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:35.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:35.227 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:11:35.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:35.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:35.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:35.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2033375415' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:35.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2033375415' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.341 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:35 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:35.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:35.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:35.548 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:35.548 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2033375415' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.548 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:35 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:11:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.557 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:35.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:35.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:35.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:35.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.827 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:35.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:35.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:35.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:35.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:35.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:36.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:11:36.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:36.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:36.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:36.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:36.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:36.379 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:36.440 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[107399]: pgmap v766: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 9489/206835 objects degraded (4.588%); 18477/206835 objects misplaced (8.933%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:36.440 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:11:36.440 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[107399]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T04:11:36.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[112165]: pgmap v766: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 9489/206835 objects degraded (4.588%); 18477/206835 objects misplaced (8.933%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:36.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:11:36.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-mon[112165]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T04:11:36.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:36.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:36 smithi012 ceph-mon[101180]: pgmap v766: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 9489/206835 objects degraded (4.588%); 18477/206835 objects misplaced (8.933%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:11:36.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:36 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:11:36.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:36 smithi012 ceph-mon[101180]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T04:11:36.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:36.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:36.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:36.797 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:36.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:36.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:36.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:36] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-09-14T04:11:37.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:37.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:37.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:37.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:11:37.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:37.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:37.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:37.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:37.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:37.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:37.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:37.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:37.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:37.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:37.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:37.897 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:37.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:37.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:37.951 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:38.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:38.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:38.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:38.179 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:38.252 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:11:38.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:38.321 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[107399]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 9612/199938 objects degraded (4.807%); 17749/199938 objects misplaced (8.877%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:11:38.321 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/750563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.322 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.322 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[112165]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 9612/199938 objects degraded (4.807%); 17749/199938 objects misplaced (8.877%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:11:38.322 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/750563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.322 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:38 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:38.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.468 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:38.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:38.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:38.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:38 smithi012 ceph-mon[101180]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 9612/199938 objects degraded (4.807%); 17749/199938 objects misplaced (8.877%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:11:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/750563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:38 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:11:38.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:38.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:38.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:38.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:38.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:38.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:39.094 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:39.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:39.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:39.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:39.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:39.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:39.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:39.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:11:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[112165]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T04:11:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9612/199938 objects degraded (4.807%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:39.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:11:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[107399]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T04:11:39.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9612/199938 objects degraded (4.807%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:39.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:39.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:39 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:11:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:11:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:39 smithi012 ceph-mon[101180]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T04:11:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9612/199938 objects degraded (4.807%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:39.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:39.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:39.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:39.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:39.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:39.943 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:39.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:40.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:40.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:40.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:40.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.367 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:40.468 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:40.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:40.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:40 smithi012 ceph-mon[101180]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 9612/197121 objects degraded (4.876%); 17749/197121 objects misplaced (9.004%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:11:40.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:40.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:40 smithi001 ceph-mon[107399]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 9612/197121 objects degraded (4.876%); 17749/197121 objects misplaced (9.004%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:11:40.714 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:40 smithi001 ceph-mon[112165]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 9612/197121 objects degraded (4.876%); 17749/197121 objects misplaced (9.004%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:11:40.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:40.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:40.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:40.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:40.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:40.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:41.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:41.136 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:41.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:41.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:41.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.307 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:41.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:41.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:41.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:41.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:41.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:41.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:41.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:41.837 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:41.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:41.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:41.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:42.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:42.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:42.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:42.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:42.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:42.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:42.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:42.329 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:42 smithi012 ceph-mon[101180]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 9702/189987 objects degraded (5.107%); 17096/189987 objects misplaced (8.999%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:42.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:42.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:42.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:42.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:42.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:42.668 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:42 smithi001 ceph-mon[112165]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 9702/189987 objects degraded (5.107%); 17096/189987 objects misplaced (8.999%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:42.668 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:42 smithi001 ceph-mon[107399]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 9702/189987 objects degraded (5.107%); 17096/189987 objects misplaced (8.999%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:42.672 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:42.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:42.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:11:42.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:42.820 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:11:42.820 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:11:42.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:42.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:42.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:42.991 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:42.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:43.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:43.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:43.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:43.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:43.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:43.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:43.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:43.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:43.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:43.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:11:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.567 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:43.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:43.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:43.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:43.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:43.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:43.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:43.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:43.963 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:43.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:44.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:44.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:44.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:44.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:44.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:44.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:44.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:44 smithi001 ceph-mon[107399]: pgmap v772: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 9702/187212 objects degraded (5.182%); 17096/187212 objects misplaced (9.132%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9702/187212 objects degraded (5.182%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:44.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:44 smithi001 ceph-mon[112165]: pgmap v772: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 9702/187212 objects degraded (5.182%); 17096/187212 objects misplaced (9.132%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:44.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9702/187212 objects degraded (5.182%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:44.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:44 smithi012 ceph-mon[101180]: pgmap v772: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 9702/187212 objects degraded (5.182%); 17096/187212 objects misplaced (9.132%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:11:44.473 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9702/187212 objects degraded (5.182%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:44.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:44.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:44.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:44.738 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:44.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:44.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:44.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:44.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:44.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:45.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:45.104 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:45.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:45.141 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:11:45.141 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:11:45.176 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:45.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:45.184 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:11:45.184 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:11:45.218 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:11:45.218 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:11:45.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:11:45.251 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:11:45.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:45.290 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:11:45.290 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:11:45.300 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:11:45.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:45.337 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:11:45.337 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:11:45.376 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:11:45.376 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:11:45.410 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:11:45.411 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:11:45.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:45.453 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2104344626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.453 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.454 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1853793491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:45.454 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2104344626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.454 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.454 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1853793491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:45.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:45.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2104344626' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:45 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:11:45.517 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1853793491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:45.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:45.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:45.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:45.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:45.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:45.895 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:45.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:45.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:46.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:46.079 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:11:46.079 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:11:46.079 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:11:46.079 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:11:46.079 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:11:46.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:46.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:46.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:46.314 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:46.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[112165]: pgmap v773: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 9702/184962 objects degraded (5.245%); 17096/184962 objects misplaced (9.243%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:11:46.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:11:46.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[112165]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T04:11:46.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[107399]: pgmap v773: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 9702/184962 objects degraded (5.245%); 17096/184962 objects misplaced (9.243%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:11:46.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:11:46.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-mon[107399]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T04:11:46.416 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:46.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:46.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:46.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:46 smithi012 ceph-mon[101180]: pgmap v773: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 9702/184962 objects degraded (5.245%); 17096/184962 objects misplaced (9.243%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:11:46.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:11:46.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:46 smithi012 ceph-mon[101180]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T04:11:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:46.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:46.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:46.842 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:46.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:11:46.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:11:46.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:46.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:46.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:46] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-09-14T04:11:46.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:47.025 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:47.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:47.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:47.230 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:47.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:47.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:47.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:47.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:47.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:47.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:47.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:47.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:47.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:47.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:47.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:47.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:47.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:47.992 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:47.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:48.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:48.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:48.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:48.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:48 smithi012 ceph-mon[101180]: pgmap v775: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9759/177939 objects degraded (5.484%); 16343/177939 objects misplaced (9.185%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:48.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:48.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:48.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:48.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:48.581 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:48 smithi001 ceph-mon[112165]: pgmap v775: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9759/177939 objects degraded (5.484%); 16343/177939 objects misplaced (9.185%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:48.582 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:48 smithi001 ceph-mon[107399]: pgmap v775: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9759/177939 objects degraded (5.484%); 16343/177939 objects misplaced (9.185%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:11:48.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:48.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:48.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:48.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:48.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:48.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:49.033 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:49.135 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:49.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:49.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:49.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:49.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9759/177939 objects degraded (5.484%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:49.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:49.423 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166779 9.d e153/135 9:b228385c:::benchmark_data_smithi001_156540_object35185:head v 153'10400, mlcod=153'10398)", 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.227345662, 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18896571700000001, 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193929+0000", 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193937+0000", 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2699999999999999e-06 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193947+0000", 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0465000000000001e-05 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193953+0000", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3790000000000003e-06 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195833+0000", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001880044 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195851+0000", 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7334999999999999e-05 2024-09-14T04:11:49.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.230956+0000", 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.035105619999999997 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382896+0000", 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.151939308 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.429 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166789 9.5 9:a97ca575:::benchmark_data_smithi001_156540_object35195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:11:49.429 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:11:49.429 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.22021958300002, 2024-09-14T04:11:49.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18248584100000001, 2024-09-14T04:11:49.429 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166789 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:11:49.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201055+0000", 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201059+0000", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.873e-06 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201072+0000", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2414e-05 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201078+0000", 2024-09-14T04:11:49.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3459999999999999e-06 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211075+0000", 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0099971409999999993 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211180+0000", 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010500799999999999 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 7", 2024-09-14T04:11:49.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211291+0000", 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111189 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382943+0000", 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17165192400000001 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383362+0000", 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041839200000000002 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383402+0000", 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0034999999999999e-05 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383542+0000", 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000140316 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166788 9.d e153/135 9:b0a806cb:::benchmark_data_smithi001_156540_object35194:head v 153'10401, mlcod=153'10398)", 2024-09-14T04:11:49.435 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.21443143900001, 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17608984799999999, 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206844+0000", 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206850+0000", 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2469999999999997e-06 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206952+0000", 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010228199999999999 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.206958+0000", 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3870000000000003e-06 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211515+0000", 2024-09-14T04:11:49.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00455749 2024-09-14T04:11:49.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211535+0000", 2024-09-14T04:11:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0063000000000001e-05 2024-09-14T04:11:49.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382904+0000", 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.171368561 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382934+0000", 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.029e-05 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166805 9.7 9:e1bea35c:::benchmark_data_smithi001_156540_object35211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.19424980600002, 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16668448999999999, 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166805 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227026+0000", 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227025+0000", 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227030+0000", 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7200000000000003e-06 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227044+0000", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3312999999999999e-05 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227051+0000", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.267e-06 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227101+0000", 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9675000000000001e-05 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227226+0000", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012531400000000001 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.227357+0000", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130882 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381943+0000", 2024-09-14T04:11:49.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15458603400000001 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382052+0000", 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108744 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393514+0000", 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011461987999999999 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393546+0000", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2799000000000002e-05 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393590+0000", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3458000000000003e-05 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393710+0000", 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120443 2024-09-14T04:11:49.448 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166810 9.7 9:e438b2a3:::benchmark_data_smithi001_156540_object35216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.18810633800001, 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17089170000000001, 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166810 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233169+0000", 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233168+0000", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233175+0000", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.844e-06 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381981+0000", 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14880580400000001 2024-09-14T04:11:49.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381992+0000", 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1167e-05 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382171+0000", 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017918400000000001 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382318+0000", 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014668999999999999 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382468+0000", 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015070599999999999 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.393751+0000", 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011282654 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.394106+0000", 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035462900000000002 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.394146+0000", 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0611000000000001e-05 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.403911+0000", 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009764633 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.403957+0000", 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5862000000000002e-05 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.404061+0000", 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104258 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178516 9.7 9:e7cca128:::benchmark_data_smithi001_156540_object46922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:11:49.456 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.10835855099998, 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0890654740000001, 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178516 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312919+0000", 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7429999999999998e-06 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312926+0000", 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6290000000000004e-06 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312931+0000", 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.13e-06 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313179+0000", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024747799999999998 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313398+0000", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219287 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313540+0000", 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014250299999999999 2024-09-14T04:11:49.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318169+0000", 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046281289999999999 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322748+0000", 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004579751 2024-09-14T04:11:49.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348314+0000", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025566169 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348324+0000", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0469999999999999e-06 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401919+0000", 2024-09-14T04:11:49.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.053594996 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401983+0000", 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4016999999999994e-05 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178520 9.8 9:1c2e04c6:::benchmark_data_smithi001_156540_object46926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.10145675500002, 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0110979040000001, 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178520 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319821+0000", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1720000000000001e-06 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319826+0000", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7339999999999996e-06 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319831+0000", 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2799999999999997e-06 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320563+0000", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073229899999999995 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320671+0000", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107984 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320794+0000", 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122669 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.332330+0000", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011536103000000001 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343578+0000", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011247823000000001 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343588+0000", 2024-09-14T04:11:49.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9939999999999995e-06 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330904+0000", 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98731630800000003 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330917+0000", 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2883e-05 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178521 9.7 9:e7156783:::benchmark_data_smithi001_156540_object46927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.10143675099999, 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069582149999999, 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178521 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.470 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319838+0000", 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319841+0000", 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.086e-06 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319846+0000", 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6110000000000001e-06 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319850+0000", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.211e-06 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322773+0000", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029226109999999999 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322906+0000", 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013334800000000001 2024-09-14T04:11:49.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323044+0000", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137897 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350098+0000", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027053395000000001 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350106+0000", 2024-09-14T04:11:49.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5750000000000001e-06 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416318+0000", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.066211786 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416396+0000", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8298000000000001e-05 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426758+0000", 2024-09-14T04:11:49.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010361354 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426797+0000", 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9487999999999999e-05 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178528 9.f 9:f3e5f4f3:::benchmark_data_smithi001_156540_object46934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:11:49.476 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.09293025099998, 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.087887475, 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178528 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328344+0000", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328349+0000", 2024-09-14T04:11:49.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0950000000000003e-06 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328361+0000", 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1915e-05 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328367+0000", 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6549999999999998e-06 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350130+0000", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021762786999999999 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350137+0000", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8630000000000002e-06 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368572+0000", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.018434893 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368729+0000", 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015681500000000001 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368871+0000", 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141779 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402186+0000", 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033314990000000003 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402204+0000", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8601e-05 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416128+0000", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013924158000000001 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416185+0000", 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6869e-05 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416233+0000", 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7602999999999998e-05 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178532 9.7 9:e0e67f4c:::benchmark_data_smithi001_156540_object46938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.08598614599998, 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.088792084, 2024-09-14T04:11:49.483 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178532 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.484 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:49.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:49.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:11:49.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:11:49.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335288+0000", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335293+0000", 2024-09-14T04:11:49.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2470000000000002e-06 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335303+0000", 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0829000000000001e-05 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335310+0000", 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9009999999999997e-06 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350158+0000", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014848175999999999 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350165+0000", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416444+0000", 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0662793960000001 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416514+0000", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9778999999999997e-05 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416588+0000", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3963999999999999e-05 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423845+0000", 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072567669999999999 2024-09-14T04:11:49.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423979+0000", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013418300000000001 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423990+0000", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1459e-05 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424029+0000", 2024-09-14T04:11:49.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.892e-05 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424082+0000", 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2386000000000003e-05 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178537 9.8 9.6a0d3bf8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:11:49.492 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.070203311, 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97989574099999999, 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178537 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351075+0000", 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.666e-06 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351081+0000", 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4840000000000001e-06 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351086+0000", 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0699999999999997e-06 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351541+0000", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045494699999999998 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351549+0000", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.498e-06 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330958+0000", 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97940985199999997 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330968+0000", 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6840000000000008e-06 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178547 9.8 9.c42478a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.04219701400001, 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.952303813, 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178547 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379078+0000", 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379083+0000", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0189999999999997e-06 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379094+0000", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1126e-05 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379099+0000", 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:11:49.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379137+0000", 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.786e-05 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0891e-05 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331371+0000", 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95222277899999996 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331382+0000", 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1686e-05 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178548 9.f 9:f6c46334:::benchmark_data_smithi001_156540_object46954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:11:49.502 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 304.04212706800001, 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0510430099999999, 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178548 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:11:49.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379152+0000", 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0289999999999999e-06 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379161+0000", 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8910000000000001e-06 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379166+0000", 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1440000000000002e-06 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379197+0000", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0839000000000001e-05 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379202+0000", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1309999999999997e-06 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420298+0000", 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410965350000001 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420384+0000", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5554000000000001e-05 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420466+0000", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2263999999999997e-05 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426757+0000", 2024-09-14T04:11:49.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006290459 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430060+0000", 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033035619999999999 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430121+0000", 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1101999999999999e-05 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430165+0000", 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4026000000000002e-05 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430192+0000", 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6142000000000001e-05 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262060 9.5 9:aa922bae:::benchmark_data_smithi001_156540_object130466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 212.938040535, 2024-09-14T04:11:49.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.199390137, 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262060 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483234+0000", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483238+0000", 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2024-09-14T04:11:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483247+0000", 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8869999999999998e-06 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483252+0000", 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.882e-06 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483300+0000", 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7879e-05 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483306+0000", 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7139999999999998e-06 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647214+0000", 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16390846000000001 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647330+0000", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011582 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647443+0000", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011323999999999999 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653349+0000", 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059055039999999998 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653931+0000", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058244600000000003 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682380+0000", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028448703999999998 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682433+0000", 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3291000000000002e-05 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682625+0000", 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019192999999999999 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262062 9.b 9:db855ab8:::benchmark_data_smithi001_156540_object130468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 212.928141428, 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26506028100000001, 2024-09-14T04:11:49.516 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262062 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493133+0000", 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493140+0000", 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7069999999999992e-06 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493155+0000", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4440999999999999e-05 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493162+0000", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6499999999999999e-06 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493204+0000", 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2209000000000002e-05 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493212+0000", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9929999999999995e-06 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654542+0000", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.161330167 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654750+0000", 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020799300000000001 2024-09-14T04:11:49.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654877+0000", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012702300000000001 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663336+0000", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084589330000000001 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663377+0000", 2024-09-14T04:11:49.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1032999999999998e-05 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757657+0000", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094279643999999996 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757711+0000", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4691000000000002e-05 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758194+0000", 2024-09-14T04:11:49.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048327800000000002 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85190 10.9 10.cff927e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 120.16364416499999, 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.954429588, 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85190 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257630+0000", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257695+0000", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4649999999999999e-05 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257714+0000", 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8902999999999998e-05 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257722+0000", 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7770000000000002e-06 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257768+0000", 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6112999999999997e-05 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257776+0000", 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.191e-06 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212048+0000", 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95427256999999999 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212061+0000", 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2573e-05 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.529 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85203 10.9 10.6280d1d9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 120.15065031899999, 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94195282000000002, 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85203 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270624+0000", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270697+0000", 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2700999999999994e-05 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270712+0000", 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4982e-05 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270719+0000", 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7500000000000003e-06 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270772+0000", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3193999999999999e-05 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270780+0000", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4719999999999998e-06 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212569+0000", 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94178903899999999 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212578+0000", 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1409999999999994e-06 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85221 10.9 10.af3d50f9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 120.13280350399999, 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92413678799999999, 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.534 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85221 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288471+0000", 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288525+0000", 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3662999999999998e-05 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288537+0000", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2326e-05 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288543+0000", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5550000000000002e-06 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288594+0000", 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0821e-05 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288602+0000", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8759999999999994e-06 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212602+0000", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92399942800000001 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212609+0000", 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.538 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85230 10.9 10.520f1ec9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 120.11289390899999, 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90426250600000002, 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85230 2024-09-14T04:11:49.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308380+0000", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308415+0000", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4962999999999997e-05 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308432+0000", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7719000000000001e-05 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308439+0000", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9399999999999996e-06 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308497+0000", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7686000000000003e-05 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308508+0000", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0671e-05 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212636+0000", 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90412800299999996 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212644+0000", 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5329999999999994e-06 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.543 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:139257 10.7 e216/202 10:eaa33927:::benchmark_data_smithi001_190881_object24117:head v 216'8792, mlcod=0'0)", 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:01.385508+0000", 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 48.035768116, 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094527067000000006, 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:49.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385508+0000", 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385508+0000", 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385507+0000", 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385511+0000", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9539999999999998e-06 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385516+0000", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3589999999999999e-06 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.385520+0000", 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6619999999999998e-06 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.399087+0000", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013567022 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.399141+0000", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4384e-05 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.479939+0000", 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080797510000000003 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:49.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:01.480035+0000", 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5613999999999996e-05 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:49.548 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:11:49.586 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:49.587 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9759/177939 objects degraded (5.484%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:49.587 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9759/177939 objects degraded (5.484%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:49.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:49.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:49.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:49.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:49.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:49.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:50.042 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:50.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:50.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:50.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:50.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:50.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:11:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:50.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:50.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:50.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:50.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:50 smithi012 ceph-mon[101180]: pgmap v776: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9759/175068 objects degraded (5.574%); 16343/175068 objects misplaced (9.335%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:11:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:50.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:50.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.664 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:50.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[107399]: pgmap v776: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9759/175068 objects degraded (5.574%); 16343/175068 objects misplaced (9.335%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[112165]: pgmap v776: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9759/175068 objects degraded (5.574%); 16343/175068 objects misplaced (9.335%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:11:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:50.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:50.796 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:50.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:50.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:50.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:50.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:50.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:51.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:51.086 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:51.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:51.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:51.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:51.325 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:51.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:51.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:51.511 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:51.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:51.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:51.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:51.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:51.809 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:51.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:51.931 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:51.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:52.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:52.070 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:52.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:52.207 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:52.309 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:52.359 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:11:52.361 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:52.367 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:11:52.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:52.496 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:52 smithi012 ceph-mon[101180]: pgmap v777: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9759/171312 objects degraded (5.697%); 16343/171312 objects misplaced (9.540%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:11:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2612088363' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:11:52.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:52.635 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:52.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[107399]: pgmap v777: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9759/171312 objects degraded (5.697%); 16343/171312 objects misplaced (9.540%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:11:52.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2612088363' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:11:52.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[112165]: pgmap v777: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9759/171312 objects degraded (5.697%); 16343/171312 objects misplaced (9.540%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:11:52.716 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2612088363' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.716 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:11:52.716 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:11:52.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:52.776 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:52.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:52.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:52.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:53.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:53.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:53.069 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:11:53.069 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:11:53.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177112 10.0 10:074c5975:::benchmark_data_smithi001_190881_object61972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.054077+0000", 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012694772, 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012727659000000001, 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177112 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054077+0000", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054077+0000", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054075+0000", 2024-09-14T04:11:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054082+0000", 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9569999999999996e-06 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054095+0000", 2024-09-14T04:11:53.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.239e-05 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054102+0000", 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3719999999999999e-06 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054167+0000", 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5396999999999997e-05 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054323+0000", 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015599099999999999 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054463+0000", 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139748 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.063616+0000", 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091531609999999999 2024-09-14T04:11:53.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.063626+0000", 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8709999999999992e-06 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177113 10.0 10:0215049c:::benchmark_data_smithi001_190881_object61973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.054140+0000", 2024-09-14T04:11:53.075 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012631850999999999, 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012780728999999999, 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177113 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054140+0000", 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054140+0000", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054139+0000", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054143+0000", 2024-09-14T04:11:53.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9790000000000004e-06 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054159+0000", 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5724000000000001e-05 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054175+0000", 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6648999999999999e-05 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.058058+0000", 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038825830000000002 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.058203+0000", 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014524399999999999 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.058344+0000", 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014098 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:53.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.063668+0000", 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053242009999999998 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066539+0000", 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028703140000000001 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.081 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177120 10.a 10:53c6a336:::benchmark_data_smithi001_190881_object61980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.055097+0000", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.01167465, 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011926541000000001, 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177120 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055097+0000", 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055097+0000", 2024-09-14T04:11:53.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055096+0000", 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055100+0000", 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2350000000000001e-06 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055112+0000", 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1786e-05 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055117+0000", 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.843e-06 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055152+0000", 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5194999999999998e-05 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055281+0000", 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012917899999999999 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.055411+0000", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129574 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.059265+0000", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038542530000000002 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:177110 10.a e229/210)", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.064563+0000", 2024-09-14T04:11:53.087 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0022087109999999999, 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025374680000000002, 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064563+0000", 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064563+0000", 2024-09-14T04:11:53.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064562+0000", 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064567+0000", 2024-09-14T04:11:53.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9679999999999997e-06 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064579+0000", 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2082999999999999e-05 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064587+0000", 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2560000000000002e-06 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.090 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:177120 10.a e229/210)", 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.064601+0000", 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0021707810000000001, 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025448229999999999, 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064601+0000", 2024-09-14T04:11:53.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064601+0000", 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064600+0000", 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064603+0000", 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9289999999999998e-06 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064609+0000", 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5430000000000001e-06 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064613+0000", 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0559999999999998e-06 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.093 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177122 10.b 10:da6442fd:::benchmark_data_smithi001_190881_object61982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.064628+0000", 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.002143673, 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025678760000000002, 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177122 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064628+0000", 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064628+0000", 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064626+0000", 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064634+0000", 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3699999999999997e-06 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064643+0000", 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9849999999999995e-06 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064648+0000", 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4149999999999998e-06 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064761+0000", 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011280100000000001 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064886+0000", 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124828 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.065010+0000", 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000123762 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177123 10.a 10.c725f60a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.064663+0000", 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0021080690000000002, 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002602605, 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.099 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177123 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064663+0000", 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064663+0000", 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064663+0000", 2024-09-14T04:11:53.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064665+0000", 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4820000000000001e-06 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064671+0000", 2024-09-14T04:11:53.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9139999999999996e-06 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064682+0000", 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0465999999999999e-05 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177124 10.f 10:f52210e0:::benchmark_data_smithi001_190881_object61984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.064687+0000", 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.002084228, 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026360369999999999, 2024-09-14T04:11:53.103 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177124 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064687+0000", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064687+0000", 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064687+0000", 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064689+0000", 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2749999999999998e-06 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064695+0000", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6330000000000001e-06 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.064699+0000", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.053e-06 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.065843+0000", 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011439779999999999 2024-09-14T04:11:53.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.065961+0000", 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117724 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066079+0000", 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118468 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:177120 10.a e229/210)", 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.066377+0000", 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00039397999999999999, 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010129939999999999, 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066377+0000", 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066377+0000", 2024-09-14T04:11:53.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066376+0000", 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066384+0000", 2024-09-14T04:11:53.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.106e-06 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066399+0000", 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4939e-05 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.066407+0000", 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7610000000000007e-06 2024-09-14T04:11:53.111 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:177110 10.a 10:51141779:::benchmark_data_smithi001_190881_object61970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:53.049147+0000", 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.017624593000000001, 2024-09-14T04:11:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018294852, 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 177110 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049147+0000", 2024-09-14T04:11:53.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049147+0000", 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049146+0000", 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049150+0000", 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049158+0000", 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5789999999999998e-06 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049162+0000", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4360000000000004e-06 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049201+0000", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8420000000000001e-05 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049278+0000", 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7824000000000006e-05 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.049349+0000", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0952999999999996e-05 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.053033+0000", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003683216 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.053694+0000", 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066098700000000003 2024-09-14T04:11:53.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:53.054262+0000", 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00056819900000000003 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 10 2024-09-14T04:11:53.118 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:11:53.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:53.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:53.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:53.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:53.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:53.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:53.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:53.626 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:53.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[112165]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:11:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:53 smithi001 ceph-mon[107399]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T04:11:53.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:53.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:53.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:53 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:11:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:11:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3624918968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:11:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:53 smithi012 ceph-mon[101180]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T04:11:53.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:53.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:53.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:54.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:54.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:54.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:54.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:54.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:54.315 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:54.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:54.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:54.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:54.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:54.698 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:54 smithi001 ceph-mon[112165]: pgmap v779: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 9823/164106 objects degraded (5.986%); 15617/164106 objects misplaced (9.516%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:11:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9823/164106 objects degraded (5.986%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:54 smithi001 ceph-mon[107399]: pgmap v779: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 9823/164106 objects degraded (5.986%); 15617/164106 objects misplaced (9.516%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:11:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9823/164106 objects degraded (5.986%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:54.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:54.831 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:11:54.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:11:54.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:54 smithi012 ceph-mon[101180]: pgmap v779: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 9823/164106 objects degraded (5.986%); 15617/164106 objects misplaced (9.516%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:11:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9823/164106 objects degraded (5.986%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:54.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:54.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:54.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:11:54.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:55.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:55.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:55.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:55.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:55.323 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:11:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:55.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:55.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:55.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:55.530 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:55.546 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:11:55.546 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:11:55.585 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:55.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:11:55.592 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:11:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.627 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:55.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:55.634 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:11:55.634 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:11:55.669 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:11:55.669 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:11:55.701 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:11:55.702 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:11:55.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:55.745 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:11:55.745 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:11:55.778 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:11:55.779 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:11:55.779 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:55.818 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:11:55.818 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:11:55.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:55.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:55.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:56.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:56.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:56.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:56.210 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:56.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:56.334 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:56.436 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:56.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:11:56.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:56.593 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:11:56.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:11:56.672 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:56 smithi001 ceph-mon[107399]: pgmap v780: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9823/161811 objects degraded (6.071%); 15617/161811 objects misplaced (9.651%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:56.672 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3256884273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:56.672 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:56 smithi001 ceph-mon[112165]: pgmap v780: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9823/161811 objects degraded (6.071%); 15617/161811 objects misplaced (9.651%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:56.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3256884273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:56.726 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:56.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:56 smithi012 ceph-mon[101180]: pgmap v780: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 9823/161811 objects degraded (6.071%); 15617/161811 objects misplaced (9.651%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3256884273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:11:56.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:56.895 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:56.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:11:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:11:56] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-09-14T04:11:56.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:57.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:57.132 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:57.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:11:57.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:57.320 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:57.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:57.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:57.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:57.592 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:57.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:57.728 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:57.830 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:57.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:57.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:57.965 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:58.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:58.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:58.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:58.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:58.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:58.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:58.315 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:11:58.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:11:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:58.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:11:58.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:58.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:58.639 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:58 smithi001 ceph-mon[112165]: pgmap v781: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9898/154539 objects degraded (6.405%); 14865/154539 objects misplaced (9.619%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:58.640 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:58 smithi001 ceph-mon[107399]: pgmap v781: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9898/154539 objects degraded (6.405%); 14865/154539 objects misplaced (9.619%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:58.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:58.704 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:11:58.705 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:11:58.714 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:58.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:58 smithi012 ceph-mon[101180]: pgmap v781: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9898/154539 objects degraded (6.405%); 14865/154539 objects misplaced (9.619%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:11:58.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:58.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:58.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:11:58.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:58.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:58.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:11:59.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:59.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:59.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:59.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:59.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:59.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:11:59.402 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:59.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:11:59.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:11:59.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:11:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:11:59.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:11:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:59.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:11:59.698 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:11:59.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:11:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9898/154539 objects degraded (6.405%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:11:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9898/154539 objects degraded (6.405%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:59.795 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:11:59.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:11:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:11:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9898/154539 objects degraded (6.405%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:11:59.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:11:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:11:59.977 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:11:59.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:00.078 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:00.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:00.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:00.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:00.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:00.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:00 smithi001 ceph-mon[107399]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:00 smithi001 ceph-mon[107399]: pgmap v782: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9898/151713 objects degraded (6.524%); 14865/151713 objects misplaced (9.798%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:00 smithi001 ceph-mon[112165]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:00 smithi001 ceph-mon[112165]: pgmap v782: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9898/151713 objects degraded (6.524%); 14865/151713 objects misplaced (9.798%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:00.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:00.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:00.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:00.672 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":781,"stamp":"2024-09-14T04:11:57.173520+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3374782461,"num_objects":51513,"num_object_clones":0,"num_object_copies":154539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9898,"num_objects_misplaced":14865,"num_objects_unfound":0,"num_objects_dirty":51513,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178853,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10835,"num_bytes_recovered":682165714,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121444,"ondisk_log_size":121444,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":318,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15232936,"kb_used_data":10724904,"kb_used_omap":33,"kb_used_meta":4507550,"kb_avail":734728280,"statfs":{"total":767960285184,"available":752361758720,"internally_reserved":0,"allocated":10982301696,"data_stored":10970760577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4615731636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,7,11,3,14,7,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-511180800,"num_objects":-7800,"num_object_clones":0,"num_object_copies":-23400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":139,"num_objects_misplaced":-1478,"num_objects_unfound":0,"num_objects_dirty":-7800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7800,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":353,"num_bytes_recovered":23134208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011943"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":247,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.390222+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:11:52.390222+0000","last_peered":"2024-09-14T04:11:52.390222+0000","last_clean":"2024-09-14T04:11:52.390222+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:11:52.390222+0000","last_undegraded":"2024-09-14T04:11:52.390222+0000","last_fullsized":"2024-09-14T04:11:52.390222+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":200,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364184+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:11:52.364184+0000","last_peered":"2024-09-14T04:11:52.364184+0000","last_clean":"2024-09-14T04:11:52.364184+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:11:52.364184+0000","last_undegraded":"2024-09-14T04:11:52.364184+0000","last_fullsized":"2024-09-14T04:11:52.364184+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306567+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:11:45.306567+0000","last_peered":"2024-09-14T04:11:45.306567+0000","last_clean":"2024-09-14T04:11:45.306567+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:11:45.306567+0000","last_undegraded":"2024-09-14T04:11:45.306567+0000","last_fullsized":"2024-09-14T04:11:45.306567+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363099+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:11:52.363099+0000","last_peered":"2024-09-14T04:11:52.363099+0000","last_clean":"2024-09-14T04:11:52.363099+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:11:52.363099+0000","last_undegraded":"2024-09-14T04:11:52.363099+0000","last_fullsized":"2024-09-14T04:11:52.363099+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.365535+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:11:52.365535+0000","last_peered":"2024-09-14T04:11:52.365535+0000","last_clean":"2024-09-14T04:11:52.365535+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:11:52.365535+0000","last_undegraded":"2024-09-14T04:11:52.365535+0000","last_fullsized":"2024-09-14T04:11:52.365535+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312266+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:11:45.312266+0000","last_peered":"2024-09-14T04:11:45.312266+0000","last_clean":"2024-09-14T04:11:45.312266+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:11:45.312266+0000","last_undegraded":"2024-09-14T04:11:45.312266+0000","last_fullsized":"2024-09-14T04:11:45.312266+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.394432+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:11:52.394432+0000","last_peered":"2024-09-14T04:11:52.394432+0000","last_clean":"2024-09-14T04:11:52.394432+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:11:52.394432+0000","last_undegraded":"2024-09-14T04:11:52.394432+0000","last_fullsized":"2024-09-14T04:11:52.394432+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":279,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310351+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:11:45.310351+0000","last_peered":"2024-09-14T04:11:45.310351+0000","last_clean":"2024-09-14T04:11:45.310351+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:11:45.310351+0000","last_undegraded":"2024-09-14T04:11:45.310351+0000","last_fullsized":"2024-09-14T04:11:45.310351+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"229'11541","reported_seq":11596,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:56.761848+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:11:56.761848+0000","last_peered":"2024-09-14T04:11:56.761848+0000","last_clean":"2024-09-14T04:11:56.761848+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:11:56.761848+0000","last_undegraded":"2024-09-14T04:11:56.761848+0000","last_fullsized":"2024-09-14T04:11:56.761848+0000","mapping_epoch":184,"log_start":"198'1800","ondisk_log_start":"198'1800","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9741,"log_dups_size":0,"ondisk_log_size":9741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11541,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":239,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313335+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:11:45.313335+0000","last_peered":"2024-09-14T04:11:45.313335+0000","last_clean":"2024-09-14T04:11:45.313335+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:11:45.313335+0000","last_undegraded":"2024-09-14T04:11:45.313335+0000","last_fullsized":"2024-09-14T04:11:45.313335+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306012+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:11:45.306012+0000","last_peered":"2024-09-14T04:11:45.306012+0000","last_clean":"2024-09-14T04:11:45.306012+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:11:45.306012+0000","last_undegraded":"2024-09-14T04:11:45.306012+0000","last_fullsized":"2024-09-14T04:11:45.306012+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"229'11413","reported_seq":17385,"reported_epoch":229,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:11:56.161982+0000","last_change":"2024-09-14T04:10:35.400317+0000","last_active":"2024-09-14T04:11:56.161982+0000","last_peered":"2024-09-14T04:11:56.161982+0000","last_clean":"2024-09-14T04:09:49.203876+0000","last_became_active":"2024-09-14T04:09:51.437466+0000","last_became_peered":"2024-09-14T04:09:51.437466+0000","last_unstale":"2024-09-14T04:11:56.161982+0000","last_undegraded":"2024-09-14T04:09:51.253655+0000","last_fullsized":"2024-09-14T04:09:51.216149+0000","mapping_epoch":202,"log_start":"199'4800","ondisk_log_start":"199'4800","created":184,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:19:14.518927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207028224,"num_objects":3159,"num_object_clones":0,"num_object_copies":9477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1769,"num_objects_misplaced":1308,"num_objects_unfound":0,"num_objects_dirty":3159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11413,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":193003520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":1308},{"shards":"3,6","objects":82}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":239,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.384213+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:11:52.384213+0000","last_peered":"2024-09-14T04:11:52.384213+0000","last_clean":"2024-09-14T04:11:52.384213+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:11:52.384213+0000","last_undegraded":"2024-09-14T04:11:52.384213+0000","last_fullsized":"2024-09-14T04:11:52.384213+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305270+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:11:45.305270+0000","last_peered":"2024-09-14T04:11:45.305270+0000","last_clean":"2024-09-14T04:11:45.305270+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:11:45.305270+0000","last_undegraded":"2024-09-14T04:11:45.305270+0000","last_fullsized":"2024-09-14T04:11:45.305270+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"229'11274","reported_seq":12563,"reported_epoch":229,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:11:54.892059+0000","last_change":"2024-09-14T04:10:36.624348+0000","last_active":"2024-09-14T04:11:54.892059+0000","last_peered":"2024-09-14T04:11:54.892059+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:11:54.892059+0000","last_undegraded":"2024-09-14T04:11:54.892059+0000","last_fullsized":"2024-09-14T04:11:54.892059+0000","mapping_epoch":210,"log_start":"198'3300","ondisk_log_start":"198'3300","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7974,"log_dups_size":0,"ondisk_log_size":7974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214171648,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7270,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11274,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":594,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-3595}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":210,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.384192+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:11:52.384192+0000","last_peered":"2024-09-14T04:11:52.384192+0000","last_clean":"2024-09-14T04:11:52.384192+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:11:52.384192+0000","last_undegraded":"2024-09-14T04:11:52.384192+0000","last_fullsized":"2024-09-14T04:11:52.384192+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":195,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310180+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:11:45.310180+0000","last_peered":"2024-09-14T04:11:45.310180+0000","last_clean":"2024-09-14T04:11:45.310180+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:11:45.310180+0000","last_undegraded":"2024-09-14T04:11:45.310180+0000","last_fullsized":"2024-09-14T04:11:45.310180+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"229'11205","reported_seq":11306,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.605880+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:11:52.605880+0000","last_peered":"2024-09-14T04:11:52.605880+0000","last_clean":"2024-09-14T04:11:52.605880+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:11:52.605880+0000","last_undegraded":"2024-09-14T04:11:52.605880+0000","last_fullsized":"2024-09-14T04:11:52.605880+0000","mapping_epoch":184,"log_start":"198'2800","ondisk_log_start":"198'2800","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8405,"log_dups_size":0,"ondisk_log_size":8405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216989696,"num_objects":3311,"num_object_clones":0,"num_object_copies":9933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11205,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":216,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:51.746576+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:11:51.746576+0000","last_peered":"2024-09-14T04:11:51.746576+0000","last_clean":"2024-09-14T04:11:51.746576+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:11:51.746576+0000","last_undegraded":"2024-09-14T04:11:51.746576+0000","last_fullsized":"2024-09-14T04:11:51.746576+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364285+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:11:52.364285+0000","last_peered":"2024-09-14T04:11:52.364285+0000","last_clean":"2024-09-14T04:11:52.364285+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:11:52.364285+0000","last_undegraded":"2024-09-14T04:11:52.364285+0000","last_fullsized":"2024-09-14T04:11:52.364285+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"229'11030","reported_seq":17901,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:54.797850+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:11:54.797850+0000","last_peered":"2024-09-14T04:11:54.797850+0000","last_clean":"2024-09-14T04:11:54.797850+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:11:54.797850+0000","last_undegraded":"2024-09-14T04:11:54.797850+0000","last_fullsized":"2024-09-14T04:11:54.797850+0000","mapping_epoch":217,"log_start":"198'3100","ondisk_log_start":"198'3100","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7930,"log_dups_size":0,"ondisk_log_size":7930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206569472,"num_objects":3152,"num_object_clones":0,"num_object_copies":9456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11030,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":580,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312063+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:11:45.312063+0000","last_peered":"2024-09-14T04:11:45.312063+0000","last_clean":"2024-09-14T04:11:45.312063+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:11:45.312063+0000","last_undegraded":"2024-09-14T04:11:45.312063+0000","last_fullsized":"2024-09-14T04:11:45.312063+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":54,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363767+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:11:52.363767+0000","last_peered":"2024-09-14T04:11:52.363767+0000","last_clean":"2024-09-14T04:11:52.363767+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:11:52.363767+0000","last_undegraded":"2024-09-14T04:11:52.363767+0000","last_fullsized":"2024-09-14T04:11:52.363767+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":215,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:50.746947+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:11:50.746947+0000","last_peered":"2024-09-14T04:11:50.746947+0000","last_clean":"2024-09-14T04:11:50.746947+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:11:50.746947+0000","last_undegraded":"2024-09-14T04:11:50.746947+0000","last_fullsized":"2024-09-14T04:11:50.746947+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"229'11026","reported_seq":11128,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.601619+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:11:52.601619+0000","last_peered":"2024-09-14T04:11:52.601619+0000","last_clean":"2024-09-14T04:11:52.601619+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:11:52.601619+0000","last_undegraded":"2024-09-14T04:11:52.601619+0000","last_fullsized":"2024-09-14T04:11:52.601619+0000","mapping_epoch":184,"log_start":"198'2600","ondisk_log_start":"198'2600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8426,"log_dups_size":0,"ondisk_log_size":8426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":449,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305933+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:11:45.305933+0000","last_peered":"2024-09-14T04:11:45.305933+0000","last_clean":"2024-09-14T04:11:45.305933+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:11:45.305933+0000","last_undegraded":"2024-09-14T04:11:45.305933+0000","last_fullsized":"2024-09-14T04:11:45.305933+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":231,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310307+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:11:45.310307+0000","last_peered":"2024-09-14T04:11:45.310307+0000","last_clean":"2024-09-14T04:11:45.310307+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:11:45.310307+0000","last_undegraded":"2024-09-14T04:11:45.310307+0000","last_fullsized":"2024-09-14T04:11:45.310307+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306047+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:11:45.306047+0000","last_peered":"2024-09-14T04:11:45.306047+0000","last_clean":"2024-09-14T04:11:45.306047+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:11:45.306047+0000","last_undegraded":"2024-09-14T04:11:45.306047+0000","last_fullsized":"2024-09-14T04:11:45.306047+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"229'11120","reported_seq":13427,"reported_epoch":229,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:11:56.123226+0000","last_change":"2024-09-14T04:11:13.382242+0000","last_active":"2024-09-14T04:11:56.123226+0000","last_peered":"2024-09-14T04:11:56.123226+0000","last_clean":"2024-09-14T04:09:49.209154+0000","last_became_active":"2024-09-14T04:11:13.358760+0000","last_became_peered":"2024-09-14T04:11:13.358760+0000","last_unstale":"2024-09-14T04:11:56.123226+0000","last_undegraded":"2024-09-14T04:11:13.259138+0000","last_fullsized":"2024-09-14T04:11:13.243084+0000","mapping_epoch":220,"log_start":"199'4500","ondisk_log_start":"199'4500","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:46.130184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206569472,"num_objects":3152,"num_object_clones":0,"num_object_copies":9456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3837,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11120,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":73203712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":1294},{"shards":"1,6,7","objects":1858}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.304818+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:11:45.304818+0000","last_peered":"2024-09-14T04:11:45.304818+0000","last_clean":"2024-09-14T04:11:45.304818+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:11:45.304818+0000","last_undegraded":"2024-09-14T04:11:45.304818+0000","last_fullsized":"2024-09-14T04:11:45.304818+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":197,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.369657+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:11:52.369657+0000","last_peered":"2024-09-14T04:11:52.369657+0000","last_clean":"2024-09-14T04:11:52.369657+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:11:52.369657+0000","last_undegraded":"2024-09-14T04:11:52.369657+0000","last_fullsized":"2024-09-14T04:11:52.369657+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"229'11239","reported_seq":11294,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:55.716236+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:11:55.716236+0000","last_peered":"2024-09-14T04:11:55.716236+0000","last_clean":"2024-09-14T04:11:55.716236+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:11:55.716236+0000","last_undegraded":"2024-09-14T04:11:55.716236+0000","last_fullsized":"2024-09-14T04:11:55.716236+0000","mapping_epoch":184,"log_start":"198'3872","ondisk_log_start":"198'3872","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208863232,"num_objects":3187,"num_object_clones":0,"num_object_copies":9561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11239,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":433,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305916+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:11:45.305916+0000","last_peered":"2024-09-14T04:11:45.305916+0000","last_clean":"2024-09-14T04:11:45.305916+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:11:45.305916+0000","last_undegraded":"2024-09-14T04:11:45.305916+0000","last_fullsized":"2024-09-14T04:11:45.305916+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389134+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:11:52.389134+0000","last_peered":"2024-09-14T04:11:52.389134+0000","last_clean":"2024-09-14T04:11:52.389134+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:11:52.389134+0000","last_undegraded":"2024-09-14T04:11:52.389134+0000","last_fullsized":"2024-09-14T04:11:52.389134+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":294,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363896+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:11:52.363896+0000","last_peered":"2024-09-14T04:11:52.363896+0000","last_clean":"2024-09-14T04:11:52.363896+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:11:52.363896+0000","last_undegraded":"2024-09-14T04:11:52.363896+0000","last_fullsized":"2024-09-14T04:11:52.363896+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"229'11246","reported_seq":11301,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:56.147999+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:11:56.147999+0000","last_peered":"2024-09-14T04:11:56.147999+0000","last_clean":"2024-09-14T04:11:56.147999+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:11:56.147999+0000","last_undegraded":"2024-09-14T04:11:56.147999+0000","last_fullsized":"2024-09-14T04:11:56.147999+0000","mapping_epoch":184,"log_start":"199'4700","ondisk_log_start":"199'4700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6546,"log_dups_size":0,"ondisk_log_size":6546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211812352,"num_objects":3232,"num_object_clones":0,"num_object_copies":9696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11246,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":598,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313500+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:11:45.313500+0000","last_peered":"2024-09-14T04:11:45.313500+0000","last_clean":"2024-09-14T04:11:45.313500+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:11:45.313500+0000","last_undegraded":"2024-09-14T04:11:45.313500+0000","last_fullsized":"2024-09-14T04:11:45.313500+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":212,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.307004+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:11:45.307004+0000","last_peered":"2024-09-14T04:11:45.307004+0000","last_clean":"2024-09-14T04:11:45.307004+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:11:45.307004+0000","last_undegraded":"2024-09-14T04:11:45.307004+0000","last_fullsized":"2024-09-14T04:11:45.307004+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":159,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306829+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:11:45.306829+0000","last_peered":"2024-09-14T04:11:45.306829+0000","last_clean":"2024-09-14T04:11:45.306829+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:11:45.306829+0000","last_undegraded":"2024-09-14T04:11:45.306829+0000","last_fullsized":"2024-09-14T04:11:45.306829+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.373985+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:11:52.373985+0000","last_peered":"2024-09-14T04:11:52.373985+0000","last_clean":"2024-09-14T04:11:52.373985+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:11:52.373985+0000","last_undegraded":"2024-09-14T04:11:52.373985+0000","last_fullsized":"2024-09-14T04:11:52.373985+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"229'10840","reported_seq":10940,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:53.632212+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:11:53.632212+0000","last_peered":"2024-09-14T04:11:53.632212+0000","last_clean":"2024-09-14T04:11:53.632212+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:11:53.632212+0000","last_undegraded":"2024-09-14T04:11:53.632212+0000","last_fullsized":"2024-09-14T04:11:53.632212+0000","mapping_epoch":184,"log_start":"199'4159","ondisk_log_start":"199'4159","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215875584,"num_objects":3294,"num_object_clones":0,"num_object_copies":9882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10840,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":477,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364461+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:11:52.364461+0000","last_peered":"2024-09-14T04:11:52.364461+0000","last_clean":"2024-09-14T04:11:52.364461+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:11:52.364461+0000","last_undegraded":"2024-09-14T04:11:52.364461+0000","last_fullsized":"2024-09-14T04:11:52.364461+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":242,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313420+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:11:45.313420+0000","last_peered":"2024-09-14T04:11:45.313420+0000","last_clean":"2024-09-14T04:11:45.313420+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:11:45.313420+0000","last_undegraded":"2024-09-14T04:11:45.313420+0000","last_fullsized":"2024-09-14T04:11:45.313420+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":273,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.372707+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:11:52.372707+0000","last_peered":"2024-09-14T04:11:52.372707+0000","last_clean":"2024-09-14T04:11:52.372707+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:11:52.372707+0000","last_undegraded":"2024-09-14T04:11:52.372707+0000","last_fullsized":"2024-09-14T04:11:52.372707+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.365612+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:11:52.365612+0000","last_peered":"2024-09-14T04:11:52.365612+0000","last_clean":"2024-09-14T04:11:52.365612+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:11:52.365612+0000","last_undegraded":"2024-09-14T04:11:52.365612+0000","last_fullsized":"2024-09-14T04:11:52.365612+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"229'11072","reported_seq":14403,"reported_epoch":229,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:11:52.603413+0000","last_change":"2024-09-14T04:10:35.403730+0000","last_active":"2024-09-14T04:11:52.603413+0000","last_peered":"2024-09-14T04:11:52.603413+0000","last_clean":"2024-09-14T04:09:49.197821+0000","last_became_active":"2024-09-14T04:09:51.414801+0000","last_became_peered":"2024-09-14T04:09:51.414801+0000","last_unstale":"2024-09-14T04:11:52.603413+0000","last_undegraded":"2024-09-14T04:10:35.401112+0000","last_fullsized":"2024-09-14T04:09:51.215769+0000","mapping_epoch":202,"log_start":"198'2700","ondisk_log_start":"198'2700","created":184,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8372,"log_dups_size":0,"ondisk_log_size":8372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220332032,"num_objects":3362,"num_object_clones":0,"num_object_copies":10086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11072,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":1466},{"shards":"2,7","objects":-930}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":460,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.311981+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:11:45.311981+0000","last_peered":"2024-09-14T04:11:45.311981+0000","last_clean":"2024-09-14T04:11:45.311981+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:11:45.311981+0000","last_undegraded":"2024-09-14T04:11:45.311981+0000","last_fullsized":"2024-09-14T04:11:45.311981+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":336,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.395046+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:11:52.395046+0000","last_peered":"2024-09-14T04:11:52.395046+0000","last_clean":"2024-09-14T04:11:52.395046+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:11:52.395046+0000","last_undegraded":"2024-09-14T04:11:52.395046+0000","last_fullsized":"2024-09-14T04:11:52.395046+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":280,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.362724+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:11:52.362724+0000","last_peered":"2024-09-14T04:11:52.362724+0000","last_clean":"2024-09-14T04:11:52.362724+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:11:52.362724+0000","last_undegraded":"2024-09-14T04:11:52.362724+0000","last_fullsized":"2024-09-14T04:11:52.362724+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":348,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363731+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:11:52.363731+0000","last_peered":"2024-09-14T04:11:52.363731+0000","last_clean":"2024-09-14T04:11:52.363731+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:11:52.363731+0000","last_undegraded":"2024-09-14T04:11:52.363731+0000","last_fullsized":"2024-09-14T04:11:52.363731+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"229'11073","reported_seq":12352,"reported_epoch":229,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:11:56.125118+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:11:56.125118+0000","last_peered":"2024-09-14T04:11:56.125118+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:11:56.125118+0000","last_undegraded":"2024-09-14T04:11:56.125118+0000","last_fullsized":"2024-09-14T04:11:56.125118+0000","mapping_epoch":210,"log_start":"199'4500","ondisk_log_start":"199'4500","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203882496,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3111,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11073,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3111}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":467,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.309950+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:11:45.309950+0000","last_peered":"2024-09-14T04:11:45.309950+0000","last_clean":"2024-09-14T04:11:45.309950+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:11:45.309950+0000","last_undegraded":"2024-09-14T04:11:45.309950+0000","last_fullsized":"2024-09-14T04:11:45.309950+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":213,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.307099+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:11:45.307099+0000","last_peered":"2024-09-14T04:11:45.307099+0000","last_clean":"2024-09-14T04:11:45.307099+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:11:45.307099+0000","last_undegraded":"2024-09-14T04:11:45.307099+0000","last_fullsized":"2024-09-14T04:11:45.307099+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":232,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:50.746929+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:11:50.746929+0000","last_peered":"2024-09-14T04:11:50.746929+0000","last_clean":"2024-09-14T04:11:50.746929+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:11:50.746929+0000","last_undegraded":"2024-09-14T04:11:50.746929+0000","last_fullsized":"2024-09-14T04:11:50.746929+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":227,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306130+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:11:45.306130+0000","last_peered":"2024-09-14T04:11:45.306130+0000","last_clean":"2024-09-14T04:11:45.306130+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:11:45.306130+0000","last_undegraded":"2024-09-14T04:11:45.306130+0000","last_fullsized":"2024-09-14T04:11:45.306130+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"229'11346","reported_seq":11401,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:56.103526+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:11:56.103526+0000","last_peered":"2024-09-14T04:11:56.103526+0000","last_clean":"2024-09-14T04:11:56.103526+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:11:56.103526+0000","last_undegraded":"2024-09-14T04:11:56.103526+0000","last_fullsized":"2024-09-14T04:11:56.103526+0000","mapping_epoch":184,"log_start":"198'3400","ondisk_log_start":"198'3400","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11346,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":441,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.318401+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:11:45.318401+0000","last_peered":"2024-09-14T04:11:45.318401+0000","last_clean":"2024-09-14T04:11:45.318401+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:11:45.318401+0000","last_undegraded":"2024-09-14T04:11:45.318401+0000","last_fullsized":"2024-09-14T04:11:45.318401+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364038+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:11:52.364038+0000","last_peered":"2024-09-14T04:11:52.364038+0000","last_clean":"2024-09-14T04:11:52.364038+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:11:52.364038+0000","last_undegraded":"2024-09-14T04:11:52.364038+0000","last_fullsized":"2024-09-14T04:11:52.364038+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":348,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.371440+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:11:52.371440+0000","last_peered":"2024-09-14T04:11:52.371440+0000","last_clean":"2024-09-14T04:11:52.371440+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:11:52.371440+0000","last_undegraded":"2024-09-14T04:11:52.371440+0000","last_fullsized":"2024-09-14T04:11:52.371440+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.368166+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:11:52.368166+0000","last_peered":"2024-09-14T04:11:52.368166+0000","last_clean":"2024-09-14T04:11:52.368166+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:11:52.368166+0000","last_undegraded":"2024-09-14T04:11:52.368166+0000","last_fullsized":"2024-09-14T04:11:52.368166+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"229'11120","reported_seq":12315,"reported_epoch":229,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:11:56.157408+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:11:56.157408+0000","last_peered":"2024-09-14T04:11:56.157408+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:11:56.157408+0000","last_undegraded":"2024-09-14T04:11:56.157408+0000","last_fullsized":"2024-09-14T04:11:56.157408+0000","mapping_epoch":210,"log_start":"199'4500","ondisk_log_start":"199'4500","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208076824,"num_objects":3176,"num_object_clones":0,"num_object_copies":9528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3176,"num_objects_unfound":0,"num_objects_dirty":3176,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11120,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":3176}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":475,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310778+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:11:45.310778+0000","last_peered":"2024-09-14T04:11:45.310778+0000","last_clean":"2024-09-14T04:11:45.310778+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:11:45.310778+0000","last_undegraded":"2024-09-14T04:11:45.310778+0000","last_fullsized":"2024-09-14T04:11:45.310778+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":279,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.371313+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:11:52.371313+0000","last_peered":"2024-09-14T04:11:52.371313+0000","last_clean":"2024-09-14T04:11:52.371313+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:11:52.371313+0000","last_undegraded":"2024-09-14T04:11:52.371313+0000","last_fullsized":"2024-09-14T04:11:52.371313+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":244,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312643+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:11:45.312643+0000","last_peered":"2024-09-14T04:11:45.312643+0000","last_clean":"2024-09-14T04:11:45.312643+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:11:45.312643+0000","last_undegraded":"2024-09-14T04:11:45.312643+0000","last_fullsized":"2024-09-14T04:11:45.312643+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364017+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:11:52.364017+0000","last_peered":"2024-09-14T04:11:52.364017+0000","last_clean":"2024-09-14T04:11:52.364017+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:11:52.364017+0000","last_undegraded":"2024-09-14T04:11:52.364017+0000","last_fullsized":"2024-09-14T04:11:52.364017+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"229'11076","reported_seq":11131,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:55.686730+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:11:55.686730+0000","last_peered":"2024-09-14T04:11:55.686730+0000","last_clean":"2024-09-14T04:11:55.686730+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:11:55.686730+0000","last_undegraded":"2024-09-14T04:11:55.686730+0000","last_fullsized":"2024-09-14T04:11:55.686730+0000","mapping_epoch":184,"log_start":"198'3712","ondisk_log_start":"198'3712","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7364,"log_dups_size":0,"ondisk_log_size":7364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206700544,"num_objects":3154,"num_object_clones":0,"num_object_copies":9462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11076,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":433,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305865+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:11:45.305865+0000","last_peered":"2024-09-14T04:11:45.305865+0000","last_clean":"2024-09-14T04:11:45.305865+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:11:45.305865+0000","last_undegraded":"2024-09-14T04:11:45.305865+0000","last_fullsized":"2024-09-14T04:11:45.305865+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":162,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313250+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:11:45.313250+0000","last_peered":"2024-09-14T04:11:45.313250+0000","last_clean":"2024-09-14T04:11:45.313250+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:11:45.313250+0000","last_undegraded":"2024-09-14T04:11:45.313250+0000","last_fullsized":"2024-09-14T04:11:45.313250+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":280,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363591+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:11:52.363591+0000","last_peered":"2024-09-14T04:11:52.363591+0000","last_clean":"2024-09-14T04:11:52.363591+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:11:52.363591+0000","last_undegraded":"2024-09-14T04:11:52.363591+0000","last_fullsized":"2024-09-14T04:11:52.363591+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":177,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313419+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:11:45.313419+0000","last_peered":"2024-09-14T04:11:45.313419+0000","last_clean":"2024-09-14T04:11:45.313419+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:11:45.313419+0000","last_undegraded":"2024-09-14T04:11:45.313419+0000","last_fullsized":"2024-09-14T04:11:45.313419+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"229'11168","reported_seq":11223,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:56.135892+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:11:56.135892+0000","last_peered":"2024-09-14T04:11:56.135892+0000","last_clean":"2024-09-14T04:11:56.135892+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:11:56.135892+0000","last_undegraded":"2024-09-14T04:11:56.135892+0000","last_fullsized":"2024-09-14T04:11:56.135892+0000","mapping_epoch":184,"log_start":"199'4600","ondisk_log_start":"199'4600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203423744,"num_objects":3104,"num_object_clones":0,"num_object_copies":9312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11168,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":505,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389618+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:11:52.389618+0000","last_peered":"2024-09-14T04:11:52.389618+0000","last_clean":"2024-09-14T04:11:52.389618+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:11:52.389618+0000","last_undegraded":"2024-09-14T04:11:52.389618+0000","last_fullsized":"2024-09-14T04:11:52.389618+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":54,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.365423+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:11:52.365423+0000","last_peered":"2024-09-14T04:11:52.365423+0000","last_clean":"2024-09-14T04:11:52.365423+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:11:52.365423+0000","last_undegraded":"2024-09-14T04:11:52.365423+0000","last_fullsized":"2024-09-14T04:11:52.365423+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":227,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.318651+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:11:45.318651+0000","last_peered":"2024-09-14T04:11:45.318651+0000","last_clean":"2024-09-14T04:11:45.318651+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:11:45.318651+0000","last_undegraded":"2024-09-14T04:11:45.318651+0000","last_fullsized":"2024-09-14T04:11:45.318651+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.738510+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:11:45.738510+0000","last_peered":"2024-09-14T04:11:45.738510+0000","last_clean":"2024-09-14T04:11:45.738510+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:11:45.738510+0000","last_undegraded":"2024-09-14T04:11:45.738510+0000","last_fullsized":"2024-09-14T04:11:45.738510+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305400+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:11:45.305400+0000","last_peered":"2024-09-14T04:11:45.305400+0000","last_clean":"2024-09-14T04:11:45.305400+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:11:45.305400+0000","last_undegraded":"2024-09-14T04:11:45.305400+0000","last_fullsized":"2024-09-14T04:11:45.305400+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306024+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:11:45.306024+0000","last_peered":"2024-09-14T04:11:45.306024+0000","last_clean":"2024-09-14T04:11:45.306024+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:11:45.306024+0000","last_undegraded":"2024-09-14T04:11:45.306024+0000","last_fullsized":"2024-09-14T04:11:45.306024+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":280,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.368054+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:11:52.368054+0000","last_peered":"2024-09-14T04:11:52.368054+0000","last_clean":"2024-09-14T04:11:52.368054+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:11:52.368054+0000","last_undegraded":"2024-09-14T04:11:52.368054+0000","last_fullsized":"2024-09-14T04:11:52.368054+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":236,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312424+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:11:45.312424+0000","last_peered":"2024-09-14T04:11:45.312424+0000","last_clean":"2024-09-14T04:11:45.312424+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:11:45.312424+0000","last_undegraded":"2024-09-14T04:11:45.312424+0000","last_fullsized":"2024-09-14T04:11:45.312424+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":197,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306267+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:11:45.306267+0000","last_peered":"2024-09-14T04:11:45.306267+0000","last_clean":"2024-09-14T04:11:45.306267+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:11:45.306267+0000","last_undegraded":"2024-09-14T04:11:45.306267+0000","last_fullsized":"2024-09-14T04:11:45.306267+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":159,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312149+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:11:45.312149+0000","last_peered":"2024-09-14T04:11:45.312149+0000","last_clean":"2024-09-14T04:11:45.312149+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:11:45.312149+0000","last_undegraded":"2024-09-14T04:11:45.312149+0000","last_fullsized":"2024-09-14T04:11:45.312149+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.362984+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:11:52.362984+0000","last_peered":"2024-09-14T04:11:52.362984+0000","last_clean":"2024-09-14T04:11:52.362984+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:11:52.362984+0000","last_undegraded":"2024-09-14T04:11:52.362984+0000","last_fullsized":"2024-09-14T04:11:52.362984+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389462+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:11:52.389462+0000","last_peered":"2024-09-14T04:11:52.389462+0000","last_clean":"2024-09-14T04:11:52.389462+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:11:52.389462+0000","last_undegraded":"2024-09-14T04:11:52.389462+0000","last_fullsized":"2024-09-14T04:11:52.389462+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:46.150931+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:11:46.150931+0000","last_peered":"2024-09-14T04:11:46.150931+0000","last_clean":"2024-09-14T04:11:46.150931+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:11:46.150931+0000","last_undegraded":"2024-09-14T04:11:46.150931+0000","last_fullsized":"2024-09-14T04:11:46.150931+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.368928+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:11:52.368928+0000","last_peered":"2024-09-14T04:11:52.368928+0000","last_clean":"2024-09-14T04:11:52.368928+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:11:52.368928+0000","last_undegraded":"2024-09-14T04:11:52.368928+0000","last_fullsized":"2024-09-14T04:11:52.368928+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.365318+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:11:52.365318+0000","last_peered":"2024-09-14T04:11:52.365318+0000","last_clean":"2024-09-14T04:11:52.365318+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:11:52.365318+0000","last_undegraded":"2024-09-14T04:11:52.365318+0000","last_fullsized":"2024-09-14T04:11:52.365318+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.362407+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:11:52.362407+0000","last_peered":"2024-09-14T04:11:52.362407+0000","last_clean":"2024-09-14T04:11:52.362407+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:11:52.362407+0000","last_undegraded":"2024-09-14T04:11:52.362407+0000","last_fullsized":"2024-09-14T04:11:52.362407+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306502+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:11:45.306502+0000","last_peered":"2024-09-14T04:11:45.306502+0000","last_clean":"2024-09-14T04:11:45.306502+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:11:45.306502+0000","last_undegraded":"2024-09-14T04:11:45.306502+0000","last_fullsized":"2024-09-14T04:11:45.306502+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":204,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363442+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:11:52.363442+0000","last_peered":"2024-09-14T04:11:52.363442+0000","last_clean":"2024-09-14T04:11:52.363442+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:11:52.363442+0000","last_undegraded":"2024-09-14T04:11:52.363442+0000","last_fullsized":"2024-09-14T04:11:52.363442+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":322,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.390144+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:11:52.390144+0000","last_peered":"2024-09-14T04:11:52.390144+0000","last_clean":"2024-09-14T04:11:52.390144+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:11:52.390144+0000","last_undegraded":"2024-09-14T04:11:52.390144+0000","last_fullsized":"2024-09-14T04:11:52.390144+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":50,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363901+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:11:52.363901+0000","last_peered":"2024-09-14T04:11:52.363901+0000","last_clean":"2024-09-14T04:11:52.363901+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:11:52.363901+0000","last_undegraded":"2024-09-14T04:11:52.363901+0000","last_fullsized":"2024-09-14T04:11:52.363901+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305227+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:11:45.305227+0000","last_peered":"2024-09-14T04:11:45.305227+0000","last_clean":"2024-09-14T04:11:45.305227+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:11:45.305227+0000","last_undegraded":"2024-09-14T04:11:45.305227+0000","last_fullsized":"2024-09-14T04:11:45.305227+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":235,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306314+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:11:45.306314+0000","last_peered":"2024-09-14T04:11:45.306314+0000","last_clean":"2024-09-14T04:11:45.306314+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:11:45.306314+0000","last_undegraded":"2024-09-14T04:11:45.306314+0000","last_fullsized":"2024-09-14T04:11:45.306314+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306910+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:11:45.306910+0000","last_peered":"2024-09-14T04:11:45.306910+0000","last_clean":"2024-09-14T04:11:45.306910+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:11:45.306910+0000","last_undegraded":"2024-09-14T04:11:45.306910+0000","last_fullsized":"2024-09-14T04:11:45.306910+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":222,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306639+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:11:45.306639+0000","last_peered":"2024-09-14T04:11:45.306639+0000","last_clean":"2024-09-14T04:11:45.306639+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:11:45.306639+0000","last_undegraded":"2024-09-14T04:11:45.306639+0000","last_fullsized":"2024-09-14T04:11:45.306639+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":284,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.395174+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:11:52.395174+0000","last_peered":"2024-09-14T04:11:52.395174+0000","last_clean":"2024-09-14T04:11:52.395174+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:11:52.395174+0000","last_undegraded":"2024-09-14T04:11:52.395174+0000","last_fullsized":"2024-09-14T04:11:52.395174+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":104,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310914+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:11:45.310914+0000","last_peered":"2024-09-14T04:11:45.310914+0000","last_clean":"2024-09-14T04:11:45.310914+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:11:45.310914+0000","last_undegraded":"2024-09-14T04:11:45.310914+0000","last_fullsized":"2024-09-14T04:11:45.310914+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306184+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:11:45.306184+0000","last_peered":"2024-09-14T04:11:45.306184+0000","last_clean":"2024-09-14T04:11:45.306184+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:11:45.306184+0000","last_undegraded":"2024-09-14T04:11:45.306184+0000","last_fullsized":"2024-09-14T04:11:45.306184+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":56,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363067+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:11:52.363067+0000","last_peered":"2024-09-14T04:11:52.363067+0000","last_clean":"2024-09-14T04:11:52.363067+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:11:52.363067+0000","last_undegraded":"2024-09-14T04:11:52.363067+0000","last_fullsized":"2024-09-14T04:11:52.363067+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3374186520,"num_objects":51487,"num_object_clones":0,"num_object_copies":154461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9898,"num_objects_misplaced":14865,"num_objects_unfound":0,"num_objects_dirty":51487,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178789,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10797,"num_bytes_recovered":681246720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10863194112,"data_stored":10863181896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119746,"ondisk_log_size":119746,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201656,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722624,"kb_used_data":947448,"kb_used_omap":1,"kb_used_meta":775166,"kb_avail":92022528,"statfs":{"total":95995035648,"available":94231068672,"internally_reserved":0,"allocated":970186752,"data_stored":968736731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":793770443},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463294,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2077952,"kb_used_data":1433208,"kb_used_omap":1,"kb_used_meta":644734,"kb_avail":91667200,"statfs":{"total":95995035648,"available":93867212800,"internally_reserved":0,"allocated":1467604992,"data_stored":1466155474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":660208071},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724930,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1824692,"kb_used_data":855084,"kb_used_omap":1,"kb_used_meta":969598,"kb_avail":91920460,"statfs":{"total":95995035648,"available":94126551040,"internally_reserved":0,"allocated":875606016,"data_stored":874164279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":992868806},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986566,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2805760,"kb_used_data":2002296,"kb_used_omap":1,"kb_used_meta":803070,"kb_avail":90939392,"statfs":{"total":95995035648,"available":93121937408,"internally_reserved":0,"allocated":2050351104,"data_stored":2048900798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":822344134},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,5,2,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748595,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1696372,"kb_used_data":1293676,"kb_used_omap":4,"kb_used_meta":402683,"kb_avail":92048780,"statfs":{"total":95995035648,"available":94257950720,"internally_reserved":0,"allocated":1324724224,"data_stored":1323282223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":412347948},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912121,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2091492,"kb_used_data":1846300,"kb_used_omap":6,"kb_used_meta":245177,"kb_avail":91653660,"statfs":{"total":95995035648,"available":93853347840,"internally_reserved":0,"allocated":1890611200,"data_stored":1889177457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":251061817},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468138,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1324904,"kb_used_data":833440,"kb_used_omap":5,"kb_used_meta":491450,"kb_avail":92420248,"statfs":{"total":95995035648,"available":94638333952,"internally_reserved":0,"allocated":853442560,"data_stored":852010788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":503245151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342253,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1689140,"kb_used_data":1513452,"kb_used_omap":10,"kb_used_meta":175669,"kb_avail":92056012,"statfs":{"total":95995035648,"available":94265356288,"internally_reserved":0,"allocated":1549774848,"data_stored":1548332827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":179885266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1520304128,"data_stored":1520304128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":846987264,"data_stored":846987264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1879310336,"data_stored":1879310336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1306923008,"data_stored":1306918936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2036731904,"data_stored":2036727832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":865402880,"data_stored":865402880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1445990400,"data_stored":1445986328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":961544192,"data_stored":961544192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:12:00.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:12:00.680 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:00 smithi012 ceph-mon[101180]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:00.681 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:00 smithi012 ceph-mon[101180]: pgmap v782: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9898/151713 objects degraded (6.524%); 14865/151713 objects misplaced (9.798%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:00.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:00.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:00.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:00.941 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:00.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:00.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:01.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:01.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:01.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:01.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:01.300 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:01.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:01.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:01.501 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:01.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:01.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:01.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:01.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:01.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:01.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:01.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:01.973 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:01.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:01.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:02.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:02.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:02.120 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:02.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.253 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:02.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:02.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:02.385 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:12:02.385 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:12:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:02.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:02.437 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:12:02.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:02.508 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:02 smithi001 ceph-mon[107399]: pgmap v783: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9898/148335 objects degraded (6.673%); 14865/148335 objects misplaced (10.021%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:02.508 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:02.509 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:02 smithi001 ceph-mon[112165]: pgmap v783: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9898/148335 objects degraded (6.673%); 14865/148335 objects misplaced (10.021%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:02.509 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:02.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:02.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:02.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:02.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:02.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:02.812 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:02 smithi012 ceph-mon[101180]: pgmap v783: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9898/148335 objects degraded (6.673%); 14865/148335 objects misplaced (10.021%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:12:02.812 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:02.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:02.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:02.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:02.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:03.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:03.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:03.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:03.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.224 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:03.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:03.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:03.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.351 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:03.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:03.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:03.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:03.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:03.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:03.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:03.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:03.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[107399]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T04:12:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:03.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:03 smithi001 ceph-mon[112165]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T04:12:03.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:03.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:03 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4206918070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:03 smithi012 ceph-mon[101180]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T04:12:03.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:03.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:03.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:03.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:04.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:04.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:04.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:04.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:04.255 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:04.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:04.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:04.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:04.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:12:04.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:04.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:04.591 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:04.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:04 smithi001 ceph-mon[112165]: pgmap v785: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9950/141252 objects degraded (7.044%); 14122/141252 objects misplaced (9.998%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9950/141252 objects degraded (7.044%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:04 smithi001 ceph-mon[107399]: pgmap v785: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9950/141252 objects degraded (7.044%); 14122/141252 objects misplaced (9.998%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9950/141252 objects degraded (7.044%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:04.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:04.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:04.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:04 smithi012 ceph-mon[101180]: pgmap v785: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9950/141252 objects degraded (7.044%); 14122/141252 objects misplaced (9.998%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:04.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9950/141252 objects degraded (7.044%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:04.853 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:12:04.888 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:04.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:04.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:05.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:05.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:05.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:05.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:05.302 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:05.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:12:05.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:05.457 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:05.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:05.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:05.590 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:05.692 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:05.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:05.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:05.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:05.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:05.954 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:12:05.954 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:12:05.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:05.985 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:12:05.985 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:12:06.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:06.021 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:12:06.021 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:12:06.057 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:12:06.057 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:12:06.093 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:12:06.093 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:12:06.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:06.136 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:12:06.137 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:12:06.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:06.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:06.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[112165]: pgmap v786: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9950/139182 objects degraded (7.149%); 14122/139182 objects misplaced (10.146%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:06.137 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/457138809' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:06.138 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:06.138 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:06.138 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[107399]: pgmap v786: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9950/139182 objects degraded (7.149%); 14122/139182 objects misplaced (10.146%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:06.138 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/457138809' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:06.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:06.175 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:12:06.175 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:12:06.229 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:12:06.229 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:12:06.259 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:06.260 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:06.260 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:05 smithi012 ceph-mon[101180]: pgmap v786: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 9950/139182 objects degraded (7.149%); 14122/139182 objects misplaced (10.146%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:12:06.260 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/457138809' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:06.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:06.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:06.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:06.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:06.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:06.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:06.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:06.686 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:06.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:06.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:06.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:06.835 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:06.838 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:12:06.852 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:06.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:06.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:06] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-09-14T04:12:06.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:07.008 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:12:07.008 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:12:07.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178517 9.1 9:8c40454a:::benchmark_data_smithi001_156540_object46923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.313240+0000", 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 321.68804186300002, 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0144453609999999, 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178517 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.009 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313240+0000", 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313240+0000", 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313238+0000", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313246+0000", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9840000000000001e-06 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313255+0000", 2024-09-14T04:12:07.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1989999999999995e-06 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313262+0000", 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9650000000000002e-06 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318591+0000", 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053281580000000004 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318737+0000", 2024-09-14T04:12:07.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014606100000000001 2024-09-14T04:12:07.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318997+0000", 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000260845 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333258+0000", 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01426096 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334359+0000", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011002989999999999 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351903+0000", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017544444999999999 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351914+0000", 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0642000000000001e-05 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327658+0000", 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97574380999999999 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327685+0000", 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7390999999999999e-05 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178526 9.1 9:8b7b33d1:::benchmark_data_smithi001_156540_object46932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.320459+0000", 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 321.68082262500002, 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.007379655, 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178526 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320459+0000", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320459+0000", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320457+0000", 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320464+0000", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.08e-06 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320476+0000", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1798e-05 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320483+0000", 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0219999999999999e-06 2024-09-14T04:12:07.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323620+0000", 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003136837 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323759+0000", 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013890199999999999 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:12:07.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323907+0000", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147668 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.345444+0000", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021537417999999999 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351941+0000", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006496908 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351948+0000", 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1539999999999996e-06 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327821+0000", 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97587255299999998 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327839+0000", 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7816e-05 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178531 9.1 9:8320b82a:::benchmark_data_smithi001_156540_object46937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.333830+0000", 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 321.66745125699998, 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99404625400000002, 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:07.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178531 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333830+0000", 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333830+0000", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333829+0000", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333835+0000", 2024-09-14T04:12:07.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1430000000000003e-06 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333847+0000", 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1865000000000001e-05 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.333854+0000", 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6719999999999998e-06 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335128+0000", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012745149999999999 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335473+0000", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034538700000000001 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335891+0000", 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041794899999999998 2024-09-14T04:12:07.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.347095+0000", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01120379 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351974+0000", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048784290000000001 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351982+0000", 2024-09-14T04:12:07.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8590000000000005e-06 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327866+0000", 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97588471700000001 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.327876+0000", 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0266e-05 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21703 10.7 10:e076e219:::benchmark_data_smithi001_190881_object21702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 180.05255382499999, 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99404368700000001, 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21703 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948727+0000", 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948757+0000", 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0284e-05 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948770+0000", 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3258000000000001e-05 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948777+0000", 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9789999999999996e-06 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948870+0000", 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2341000000000002e-05 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948884+0000", 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4677e-05 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939652+0000", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99076829799999999 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939754+0000", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101926 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939848+0000", 2024-09-14T04:12:07.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3722999999999997e-05 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942664+0000", 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028159140000000001 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942684+0000", 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9796000000000002e-05 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942706+0000", 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2467e-05 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942734+0000", 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8016000000000001e-05 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942771+0000", 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7021999999999999e-05 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21712 10.7 10:e76dfa44:::benchmark_data_smithi001_190881_object21711:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:06.956516+0000", 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 180.04476577599999, 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98648943899999997, 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21712 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956516+0000", 2024-09-14T04:12:07.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956516+0000", 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956515+0000", 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956564+0000", 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9577999999999999e-05 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956577+0000", 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3285000000000001e-05 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956585+0000", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.486e-06 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956680+0000", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5290000000000004e-05 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.956695+0000", 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4929000000000001e-05 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.940952+0000", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98425708999999995 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.941091+0000", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013856099999999999 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.941224+0000", 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013316700000000001 2024-09-14T04:12:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942858+0000", 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016339270000000001 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942893+0000", 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5203999999999997e-05 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942902+0000", 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3470000000000005e-06 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942945+0000", 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2710999999999998e-05 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.943005+0000", 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9951999999999998e-05 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21874 10.a 10:50ff48f0:::benchmark_data_smithi001_190881_object21873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.050707292, 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99320812599999997, 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21874 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.045 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950573+0000", 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950634+0000", 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1093999999999995e-05 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950645+0000", 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0852999999999999e-05 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950652+0000", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8260000000000004e-06 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950713+0000", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0912999999999998e-05 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950723+0000", 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0582e-05 2024-09-14T04:12:07.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.939710+0000", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98898640699999996 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.939878+0000", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168376 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940037+0000", 2024-09-14T04:12:07.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015902599999999999 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943506+0000", 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034692540000000002 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943622+0000", 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011597800000000001 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943688+0000", 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5625999999999998e-05 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943745+0000", 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6786999999999997e-05 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943782+0000", 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7608000000000003e-05 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21881 10.a 10:594b51ff:::benchmark_data_smithi001_190881_object21880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.955093+0000", 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.04618817900001, 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98893604099999999, 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21881 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.052 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955093+0000", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955093+0000", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955092+0000", 2024-09-14T04:12:07.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955147+0000", 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4957999999999998e-05 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955163+0000", 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5698000000000001e-05 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955180+0000", 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.753e-05 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955229+0000", 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9391000000000002e-05 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955239+0000", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2310000000000002e-06 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941419+0000", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98618054200000005 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941636+0000", 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021658699999999999 2024-09-14T04:12:07.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941783+0000", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014756099999999999 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943801+0000", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002017568 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943868+0000", 2024-09-14T04:12:07.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7244999999999997e-05 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943949+0000", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0890999999999995e-05 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943993+0000", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4357000000000002e-05 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944029+0000", 2024-09-14T04:12:07.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5921999999999999e-05 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21883 10.a 10:514bc832:::benchmark_data_smithi001_190881_object21882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.045288887, 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98944681199999995, 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21883 2024-09-14T04:12:07.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955992+0000", 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956024+0000", 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2286000000000002e-05 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956039+0000", 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4662e-05 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956047+0000", 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7879999999999995e-06 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956099+0000", 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1857000000000002e-05 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956109+0000", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6940000000000007e-06 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942527+0000", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98641796100000001 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942650+0000", 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000123328 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942843+0000", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019251899999999999 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944044+0000", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001201559 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945324+0000", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012796439999999999 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945359+0000", 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4938999999999999e-05 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945401+0000", 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1993000000000003e-05 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945439+0000", 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8507999999999998e-05 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.065 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21887 10.a 10:5d6c9b5d:::benchmark_data_smithi001_190881_object21886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.04252792700001, 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99076259499999997, 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21887 2024-09-14T04:12:07.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958749+0000", 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958794+0000", 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4885000000000001e-05 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958805+0000", 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.093e-05 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958812+0000", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.255e-06 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958865+0000", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2920999999999997e-05 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958874+0000", 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2730000000000008e-06 2024-09-14T04:12:07.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944101+0000", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98522702500000003 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944185+0000", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3516000000000005e-05 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944317+0000", 2024-09-14T04:12:07.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132807 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947650+0000", 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033321980000000002 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947717+0000", 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7151999999999997e-05 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949433+0000", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001715928 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949477+0000", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4487999999999998e-05 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949516+0000", 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8992000000000002e-05 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22006 10.d 10:b8ac2021:::benchmark_data_smithi001_190881_object22005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 178.05747216399999, 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0147531949999999, 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22006 2024-09-14T04:12:07.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943808+0000", 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943951+0000", 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014249799999999999 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943965+0000", 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4518999999999999e-05 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943975+0000", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9129999999999998e-06 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944044+0000", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8853e-05 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944053+0000", 2024-09-14T04:12:07.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1870000000000003e-06 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.954838+0000", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0107845520000001 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.954995+0000", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015683799999999999 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.955105+0000", 2024-09-14T04:12:07.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110714 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958327+0000", 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003222043 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958339+0000", 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1308000000000001e-05 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958465+0000", 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012594599999999999 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958518+0000", 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3028000000000001e-05 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958563+0000", 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4975000000000002e-05 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.079 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:07.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22019 10.d 10:bd024054:::benchmark_data_smithi001_190881_object22018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 178.050142956, 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.00765544, 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22019 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.082 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951137+0000", 2024-09-14T04:12:07.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951170+0000", 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2492e-05 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951530+0000", 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036076700000000001 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951539+0000", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6500000000000002e-06 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951655+0000", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115574 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951698+0000", 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3602e-05 2024-09-14T04:12:07.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956415+0000", 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.004716731 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956562+0000", 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014654499999999999 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956725+0000", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000163361 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958620+0000", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001894939 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958629+0000", 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8559999999999996e-06 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958693+0000", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4189000000000005e-05 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958749+0000", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5680999999999998e-05 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958794+0000", 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5371000000000003e-05 2024-09-14T04:12:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85152 10.0 10:04bd257d:::benchmark_data_smithi001_190881_object85151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.78619581999999, 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2469428599999999, 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85152 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215084+0000", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215117+0000", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3624000000000002e-05 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215134+0000", 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6758000000000001e-05 2024-09-14T04:12:07.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215143+0000", 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5080000000000005e-06 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215208+0000", 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4907999999999996e-05 2024-09-14T04:12:07.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215356+0000", 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148114 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215492+0000", 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013663999999999999 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.224934+0000", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094411930000000005 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.224947+0000", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3093e-05 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215613+0000", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99066625600000002 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215623+0000", 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0377000000000001e-05 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.446326+0000", 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.230702956 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.446898+0000", 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00057135599999999997 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461915+0000", 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015017536 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462029+0000", 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011345000000000001 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.096 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85170 10.0 10:0452c479:::benchmark_data_smithi001_190881_object85169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.75789768499999, 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2236046439999999, 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85170 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243379+0000", 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243425+0000", 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5998999999999998e-05 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243438+0000", 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3225e-05 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243446+0000", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7610000000000007e-06 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243527+0000", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1230999999999995e-05 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243540+0000", 2024-09-14T04:12:07.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2397e-05 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215644+0000", 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97210399800000002 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215650+0000", 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3430000000000001e-06 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449443+0000", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2337928069999999 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449597+0000", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015371499999999999 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449766+0000", 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016950299999999999 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462179+0000", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012412929 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462187+0000", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5109999999999997e-06 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462254+0000", 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7372999999999993e-05 2024-09-14T04:12:07.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462358+0000", 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103946 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.466988+0000", 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046306300000000002 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85185 10.7 10:e9b0eeb6:::benchmark_data_smithi001_190881_object85184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.74681635799999, 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2214390960000001, 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85185 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254507+0000", 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2392999999999999e-05 2024-09-14T04:12:07.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254517+0000", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0251999999999999e-05 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254524+0000", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4910000000000002e-06 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254566+0000", 2024-09-14T04:12:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2171e-05 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254574+0000", 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1370000000000002e-06 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214574+0000", 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95999957999999996 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214587+0000", 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3329999999999999e-05 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449262+0000", 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.234674703 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449425+0000", 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016336700000000001 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:12:07.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449533+0000", 2024-09-14T04:12:07.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108441 2024-09-14T04:12:07.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458502+0000", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089684389999999999 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458552+0000", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0621999999999997e-05 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471592+0000", 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013039258 2024-09-14T04:12:07.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471645+0000", 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3643000000000001e-05 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475904+0000", 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042588529999999999 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85187 10.7 10:e711136f:::benchmark_data_smithi001_190881_object85186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.74409770599999, 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2358304539999998, 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85187 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257182+0000", 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257217+0000", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4579999999999998e-05 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257237+0000", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9930999999999999e-05 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257245+0000", 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3510000000000002e-06 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257373+0000", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012805 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257384+0000", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0361e-05 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214610+0000", 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95722658400000005 2024-09-14T04:12:07.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214618+0000", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5800000000000003e-06 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455570+0000", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2409521269999999 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455741+0000", 2024-09-14T04:12:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017140699999999999 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455928+0000", 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018640099999999999 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464920+0000", 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089924170000000008 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464981+0000", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0770999999999998e-05 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.489152+0000", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024170921000000001 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.489282+0000", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013016699999999999 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493014+0000", 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003732366 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85193 10.7 10:e361fc0b:::benchmark_data_smithi001_190881_object85192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.74151907000001, 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2371485259999999, 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.123 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85193 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259761+0000", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259817+0000", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5583e-05 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259827+0000", 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8849999999999994e-06 2024-09-14T04:12:07.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259833+0000", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8499999999999999e-06 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259874+0000", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1316000000000002e-05 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259880+0000", 2024-09-14T04:12:07.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.032e-06 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214639+0000", 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95475853099999997 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214647+0000", 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9829999999999996e-06 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458633+0000", 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2439862399999999 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458791+0000", 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157637 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458961+0000", 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000170056 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464971+0000", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006010329 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464986+0000", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5319999999999999e-05 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493078+0000", 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02809124 2024-09-14T04:12:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493147+0000", 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9715000000000006e-05 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496911+0000", 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037637360000000002 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85196 10.7 10:e2c2e82c:::benchmark_data_smithi001_190881_object85195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.738736206, 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2380154719999998, 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85196 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262544+0000", 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262604+0000", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9871000000000003e-05 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262614+0000", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0040999999999999e-05 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262621+0000", 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9129999999999997e-06 2024-09-14T04:12:07.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262656+0000", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5082000000000001e-05 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262662+0000", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1210000000000003e-06 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214668+0000", 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95200554699999995 2024-09-14T04:12:07.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214684+0000", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6385e-05 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461526+0000", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2468417430000001 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461711+0000", 2024-09-14T04:12:07.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018453799999999999 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462421+0000", 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00071004800000000002 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471513+0000", 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090922599999999996 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475962+0000", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044492680000000001 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496962+0000", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020999752 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496999+0000", 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6591000000000003e-05 2024-09-14T04:12:07.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500561+0000", 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003562164 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85205 10.7 10:ecddaef8:::benchmark_data_smithi001_190881_object85204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.73050329399999, 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2339501529999999, 2024-09-14T04:12:07.138 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85205 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270814+0000", 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6078999999999998e-05 2024-09-14T04:12:07.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270824+0000", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0349999999999999e-05 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270830+0000", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.331e-06 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270879+0000", 2024-09-14T04:12:07.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9149e-05 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270888+0000", 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4330000000000004e-06 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214707+0000", 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94381939299999995 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214715+0000", 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8250000000000005e-06 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465015+0000", 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2502998060000001 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465190+0000", 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017487000000000001 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465364+0000", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017457099999999999 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475998+0000", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01063412 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.476064+0000", 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5627e-05 2024-09-14T04:12:07.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500619+0000", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024554630000000001 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500671+0000", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2207999999999999e-05 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.504728+0000", 2024-09-14T04:12:07.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040574410000000002 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85212 10.0 10:05efc25a:::benchmark_data_smithi001_190881_object85211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.72284399099999, 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.207828165, 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85212 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278436+0000", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278496+0000", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9991999999999998e-05 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278509+0000", 2024-09-14T04:12:07.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2668e-05 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278517+0000", 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1019999999999997e-06 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278572+0000", 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5083000000000001e-05 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278583+0000", 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0424e-05 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215667+0000", 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93708416800000005 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215684+0000", 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6939000000000002e-05 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467030+0000", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2513462829999999 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467188+0000", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157966 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467374+0000", 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018620299999999999 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.477554+0000", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010179911999999999 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482157+0000", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046033660000000002 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482229+0000", 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1594000000000006e-05 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482410+0000", 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018091900000000001 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.486266+0000", 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038557079999999998 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85226 10.0 10:083bf9b5:::benchmark_data_smithi001_190881_object85225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.70434242300001, 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2170019399999998, 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:07.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85226 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:07.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296938+0000", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296984+0000", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6419999999999999e-05 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296999+0000", 2024-09-14T04:12:07.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4878999999999999e-05 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297007+0000", 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6459999999999993e-06 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297071+0000", 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4008000000000001e-05 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297083+0000", 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2167999999999999e-05 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215705+0000", 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91862224100000001 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215712+0000", 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6080000000000001e-06 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470415+0000", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2547035580000001 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470574+0000", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015836499999999999 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470739+0000", 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000165663 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482192+0000", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011453029999999999 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.490246+0000", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.008053859 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.511198+0000", 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020951908000000002 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.511262+0000", 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4046000000000003e-05 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.513941+0000", 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026789460000000002 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.161 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.162 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.162 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:07.162 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:07.162 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:12:07.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:07.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:07.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:07.352 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:07.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:07.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:07.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:07.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:07.704 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":786,"stamp":"2024-09-14T04:12:05.177240+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3039303677,"num_objects":46394,"num_object_clones":0,"num_object_copies":139182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9950,"num_objects_misplaced":14122,"num_objects_unfound":0,"num_objects_dirty":46394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183972,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11030,"num_bytes_recovered":694945234,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121463,"ondisk_log_size":121463,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14265824,"kb_used_data":9737624,"kb_used_omap":33,"kb_used_meta":4527966,"kb_avail":735695392,"statfs":{"total":767960285184,"available":753352081408,"internally_reserved":0,"allocated":9971326976,"data_stored":9959762573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4636637620},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-494338048,"num_objects":-7543,"num_object_clones":0,"num_object_copies":-22629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":127,"num_objects_misplaced":-1495,"num_objects_unfound":0,"num_objects_dirty":-7543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7543,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":378,"num_bytes_recovered":24772608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004707"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":250,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.449073+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:12:02.449073+0000","last_peered":"2024-09-14T04:12:02.449073+0000","last_clean":"2024-09-14T04:12:02.449073+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:12:02.449073+0000","last_undegraded":"2024-09-14T04:12:02.449073+0000","last_fullsized":"2024-09-14T04:12:02.449073+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":202,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.449649+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:12:02.449649+0000","last_peered":"2024-09-14T04:12:02.449649+0000","last_clean":"2024-09-14T04:12:02.449649+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:12:02.449649+0000","last_undegraded":"2024-09-14T04:12:02.449649+0000","last_fullsized":"2024-09-14T04:12:02.449649+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306567+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:11:45.306567+0000","last_peered":"2024-09-14T04:11:45.306567+0000","last_clean":"2024-09-14T04:11:45.306567+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:11:45.306567+0000","last_undegraded":"2024-09-14T04:11:45.306567+0000","last_fullsized":"2024-09-14T04:11:45.306567+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446351+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:12:02.446351+0000","last_peered":"2024-09-14T04:12:02.446351+0000","last_clean":"2024-09-14T04:12:02.446351+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:12:02.446351+0000","last_undegraded":"2024-09-14T04:12:02.446351+0000","last_fullsized":"2024-09-14T04:12:02.446351+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451899+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:12:02.451899+0000","last_peered":"2024-09-14T04:12:02.451899+0000","last_clean":"2024-09-14T04:12:02.451899+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:12:02.451899+0000","last_undegraded":"2024-09-14T04:12:02.451899+0000","last_fullsized":"2024-09-14T04:12:02.451899+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312266+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:11:45.312266+0000","last_peered":"2024-09-14T04:11:45.312266+0000","last_clean":"2024-09-14T04:11:45.312266+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:11:45.312266+0000","last_undegraded":"2024-09-14T04:11:45.312266+0000","last_fullsized":"2024-09-14T04:11:45.312266+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.394432+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:11:52.394432+0000","last_peered":"2024-09-14T04:11:52.394432+0000","last_clean":"2024-09-14T04:11:52.394432+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:11:52.394432+0000","last_undegraded":"2024-09-14T04:11:52.394432+0000","last_fullsized":"2024-09-14T04:11:52.394432+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":279,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310351+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:11:45.310351+0000","last_peered":"2024-09-14T04:11:45.310351+0000","last_clean":"2024-09-14T04:11:45.310351+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:11:45.310351+0000","last_undegraded":"2024-09-14T04:11:45.310351+0000","last_fullsized":"2024-09-14T04:11:45.310351+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"229'11781","reported_seq":11836,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:01.755928+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:12:01.755928+0000","last_peered":"2024-09-14T04:12:01.755928+0000","last_clean":"2024-09-14T04:12:01.755928+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:12:01.755928+0000","last_undegraded":"2024-09-14T04:12:01.755928+0000","last_fullsized":"2024-09-14T04:12:01.755928+0000","mapping_epoch":184,"log_start":"198'2100","ondisk_log_start":"198'2100","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9681,"log_dups_size":0,"ondisk_log_size":9681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198770688,"num_objects":3033,"num_object_clones":0,"num_object_copies":9099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11781,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":240,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.384298+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:11:52.384298+0000","last_peered":"2024-09-14T04:11:52.384298+0000","last_clean":"2024-09-14T04:11:52.384298+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:11:52.384298+0000","last_undegraded":"2024-09-14T04:11:52.384298+0000","last_fullsized":"2024-09-14T04:11:52.384298+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.363394+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:11:52.363394+0000","last_peered":"2024-09-14T04:11:52.363394+0000","last_clean":"2024-09-14T04:11:52.363394+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:11:52.363394+0000","last_undegraded":"2024-09-14T04:11:52.363394+0000","last_fullsized":"2024-09-14T04:11:52.363394+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"229'11655","reported_seq":18017,"reported_epoch":229,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:01.185997+0000","last_change":"2024-09-14T04:10:35.400317+0000","last_active":"2024-09-14T04:12:01.185997+0000","last_peered":"2024-09-14T04:12:01.185997+0000","last_clean":"2024-09-14T04:09:49.203876+0000","last_became_active":"2024-09-14T04:09:51.437466+0000","last_became_peered":"2024-09-14T04:09:51.437466+0000","last_unstale":"2024-09-14T04:12:01.185997+0000","last_undegraded":"2024-09-14T04:09:51.253655+0000","last_fullsized":"2024-09-14T04:09:51.216149+0000","mapping_epoch":202,"log_start":"199'5100","ondisk_log_start":"199'5100","created":184,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:19:14.518927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1821,"num_objects_misplaced":1061,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11655,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3140,"num_bytes_recovered":205783040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":1061},{"shards":"3,6","objects":35}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":239,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.384213+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:11:52.384213+0000","last_peered":"2024-09-14T04:11:52.384213+0000","last_clean":"2024-09-14T04:11:52.384213+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:11:52.384213+0000","last_undegraded":"2024-09-14T04:11:52.384213+0000","last_fullsized":"2024-09-14T04:11:52.384213+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305270+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:11:45.305270+0000","last_peered":"2024-09-14T04:11:45.305270+0000","last_clean":"2024-09-14T04:11:45.305270+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:11:45.305270+0000","last_undegraded":"2024-09-14T04:11:45.305270+0000","last_fullsized":"2024-09-14T04:11:45.305270+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"230'11733","reported_seq":13025,"reported_epoch":230,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:12:04.871055+0000","last_change":"2024-09-14T04:10:36.624348+0000","last_active":"2024-09-14T04:12:04.871055+0000","last_peered":"2024-09-14T04:12:04.871055+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:12:04.871055+0000","last_undegraded":"2024-09-14T04:12:04.871055+0000","last_fullsized":"2024-09-14T04:12:04.871055+0000","mapping_epoch":210,"log_start":"198'3800","ondisk_log_start":"198'3800","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7933,"log_dups_size":0,"ondisk_log_size":7933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184090624,"num_objects":2809,"num_object_clones":0,"num_object_copies":8427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7270,"num_objects_unfound":0,"num_objects_dirty":2809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11733,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":594,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-4054}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":210,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.384192+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:11:52.384192+0000","last_peered":"2024-09-14T04:11:52.384192+0000","last_clean":"2024-09-14T04:11:52.384192+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:11:52.384192+0000","last_undegraded":"2024-09-14T04:11:52.384192+0000","last_fullsized":"2024-09-14T04:11:52.384192+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":195,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310180+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:11:45.310180+0000","last_peered":"2024-09-14T04:11:45.310180+0000","last_clean":"2024-09-14T04:11:45.310180+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:11:45.310180+0000","last_undegraded":"2024-09-14T04:11:45.310180+0000","last_fullsized":"2024-09-14T04:11:45.310180+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"230'11676","reported_seq":11780,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.610227+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:12:02.610227+0000","last_peered":"2024-09-14T04:12:02.610227+0000","last_clean":"2024-09-14T04:12:02.610227+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:12:02.610227+0000","last_undegraded":"2024-09-14T04:12:02.610227+0000","last_fullsized":"2024-09-14T04:12:02.610227+0000","mapping_epoch":184,"log_start":"198'3300","ondisk_log_start":"198'3300","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8376,"log_dups_size":0,"ondisk_log_size":8376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11676,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":216,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:51.746576+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:11:51.746576+0000","last_peered":"2024-09-14T04:11:51.746576+0000","last_clean":"2024-09-14T04:11:51.746576+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:11:51.746576+0000","last_undegraded":"2024-09-14T04:11:51.746576+0000","last_fullsized":"2024-09-14T04:11:51.746576+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.449757+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:12:02.449757+0000","last_peered":"2024-09-14T04:12:02.449757+0000","last_clean":"2024-09-14T04:12:02.449757+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:12:02.449757+0000","last_undegraded":"2024-09-14T04:12:02.449757+0000","last_fullsized":"2024-09-14T04:12:02.449757+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"230'11486","reported_seq":18358,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:04.896801+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:12:04.896801+0000","last_peered":"2024-09-14T04:12:04.896801+0000","last_clean":"2024-09-14T04:12:04.896801+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:12:04.896801+0000","last_undegraded":"2024-09-14T04:12:04.896801+0000","last_fullsized":"2024-09-14T04:12:04.896801+0000","mapping_epoch":217,"log_start":"198'3500","ondisk_log_start":"198'3500","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7986,"log_dups_size":0,"ondisk_log_size":7986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176685056,"num_objects":2696,"num_object_clones":0,"num_object_copies":8088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11486,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":580,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312063+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:11:45.312063+0000","last_peered":"2024-09-14T04:11:45.312063+0000","last_clean":"2024-09-14T04:11:45.312063+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:11:45.312063+0000","last_undegraded":"2024-09-14T04:11:45.312063+0000","last_fullsized":"2024-09-14T04:11:45.312063+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":56,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443392+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:12:02.443392+0000","last_peered":"2024-09-14T04:12:02.443392+0000","last_clean":"2024-09-14T04:12:02.443392+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:12:02.443392+0000","last_undegraded":"2024-09-14T04:12:02.443392+0000","last_fullsized":"2024-09-14T04:12:02.443392+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":215,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:50.746947+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:11:50.746947+0000","last_peered":"2024-09-14T04:11:50.746947+0000","last_clean":"2024-09-14T04:11:50.746947+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:11:50.746947+0000","last_undegraded":"2024-09-14T04:11:50.746947+0000","last_fullsized":"2024-09-14T04:11:50.746947+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"230'11480","reported_seq":11584,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.563543+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:12:02.563543+0000","last_peered":"2024-09-14T04:12:02.563543+0000","last_clean":"2024-09-14T04:12:02.563543+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:12:02.563543+0000","last_undegraded":"2024-09-14T04:12:02.563543+0000","last_fullsized":"2024-09-14T04:12:02.563543+0000","mapping_epoch":184,"log_start":"198'3100","ondisk_log_start":"198'3100","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8380,"log_dups_size":0,"ondisk_log_size":8380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186908672,"num_objects":2852,"num_object_clones":0,"num_object_copies":8556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11480,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":449,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305933+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:11:45.305933+0000","last_peered":"2024-09-14T04:11:45.305933+0000","last_clean":"2024-09-14T04:11:45.305933+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:11:45.305933+0000","last_undegraded":"2024-09-14T04:11:45.305933+0000","last_fullsized":"2024-09-14T04:11:45.305933+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":231,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310307+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:11:45.310307+0000","last_peered":"2024-09-14T04:11:45.310307+0000","last_clean":"2024-09-14T04:11:45.310307+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:11:45.310307+0000","last_undegraded":"2024-09-14T04:11:45.310307+0000","last_fullsized":"2024-09-14T04:11:45.310307+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306047+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:11:45.306047+0000","last_peered":"2024-09-14T04:11:45.306047+0000","last_clean":"2024-09-14T04:11:45.306047+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:11:45.306047+0000","last_undegraded":"2024-09-14T04:11:45.306047+0000","last_fullsized":"2024-09-14T04:11:45.306047+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"229'11351","reported_seq":13653,"reported_epoch":229,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:01.183865+0000","last_change":"2024-09-14T04:11:13.382242+0000","last_active":"2024-09-14T04:12:01.183865+0000","last_peered":"2024-09-14T04:12:01.183865+0000","last_clean":"2024-09-14T04:09:49.209154+0000","last_became_active":"2024-09-14T04:11:13.358760+0000","last_became_peered":"2024-09-14T04:11:13.358760+0000","last_unstale":"2024-09-14T04:12:01.183865+0000","last_undegraded":"2024-09-14T04:11:13.259138+0000","last_fullsized":"2024-09-14T04:11:13.243084+0000","mapping_epoch":220,"log_start":"199'4800","ondisk_log_start":"199'4800","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:46.130184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191430656,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3837,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11351,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":73203712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":1063},{"shards":"1,6,7","objects":1858}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.304818+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:11:45.304818+0000","last_peered":"2024-09-14T04:11:45.304818+0000","last_clean":"2024-09-14T04:11:45.304818+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:11:45.304818+0000","last_undegraded":"2024-09-14T04:11:45.304818+0000","last_fullsized":"2024-09-14T04:11:45.304818+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":197,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.369657+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:11:52.369657+0000","last_peered":"2024-09-14T04:11:52.369657+0000","last_clean":"2024-09-14T04:11:52.369657+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:11:52.369657+0000","last_undegraded":"2024-09-14T04:11:52.369657+0000","last_fullsized":"2024-09-14T04:11:52.369657+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"229'11463","reported_seq":11518,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:00.673389+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:12:00.673389+0000","last_peered":"2024-09-14T04:12:00.673389+0000","last_clean":"2024-09-14T04:12:00.673389+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:12:00.673389+0000","last_undegraded":"2024-09-14T04:12:00.673389+0000","last_fullsized":"2024-09-14T04:12:00.673389+0000","mapping_epoch":184,"log_start":"199'4072","ondisk_log_start":"199'4072","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194183168,"num_objects":2963,"num_object_clones":0,"num_object_copies":8889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11463,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":433,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305916+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:11:45.305916+0000","last_peered":"2024-09-14T04:11:45.305916+0000","last_clean":"2024-09-14T04:11:45.305916+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:11:45.305916+0000","last_undegraded":"2024-09-14T04:11:45.305916+0000","last_fullsized":"2024-09-14T04:11:45.305916+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389134+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:11:52.389134+0000","last_peered":"2024-09-14T04:11:52.389134+0000","last_clean":"2024-09-14T04:11:52.389134+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:11:52.389134+0000","last_undegraded":"2024-09-14T04:11:52.389134+0000","last_fullsized":"2024-09-14T04:11:52.389134+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":296,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448758+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:12:02.448758+0000","last_peered":"2024-09-14T04:12:02.448758+0000","last_clean":"2024-09-14T04:12:02.448758+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:12:02.448758+0000","last_undegraded":"2024-09-14T04:12:02.448758+0000","last_fullsized":"2024-09-14T04:12:02.448758+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"229'11479","reported_seq":11534,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:01.192722+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:12:01.192722+0000","last_peered":"2024-09-14T04:12:01.192722+0000","last_clean":"2024-09-14T04:12:01.192722+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:12:01.192722+0000","last_undegraded":"2024-09-14T04:12:01.192722+0000","last_fullsized":"2024-09-14T04:12:01.192722+0000","mapping_epoch":184,"log_start":"199'4900","ondisk_log_start":"199'4900","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196542464,"num_objects":2999,"num_object_clones":0,"num_object_copies":8997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11479,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":598,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313500+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:11:45.313500+0000","last_peered":"2024-09-14T04:11:45.313500+0000","last_clean":"2024-09-14T04:11:45.313500+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:11:45.313500+0000","last_undegraded":"2024-09-14T04:11:45.313500+0000","last_fullsized":"2024-09-14T04:11:45.313500+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":212,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.307004+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:11:45.307004+0000","last_peered":"2024-09-14T04:11:45.307004+0000","last_clean":"2024-09-14T04:11:45.307004+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:11:45.307004+0000","last_undegraded":"2024-09-14T04:11:45.307004+0000","last_fullsized":"2024-09-14T04:11:45.307004+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":159,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306829+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:11:45.306829+0000","last_peered":"2024-09-14T04:11:45.306829+0000","last_clean":"2024-09-14T04:11:45.306829+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:11:45.306829+0000","last_undegraded":"2024-09-14T04:11:45.306829+0000","last_fullsized":"2024-09-14T04:11:45.306829+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.373985+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:11:52.373985+0000","last_peered":"2024-09-14T04:11:52.373985+0000","last_clean":"2024-09-14T04:11:52.373985+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:11:52.373985+0000","last_undegraded":"2024-09-14T04:11:52.373985+0000","last_fullsized":"2024-09-14T04:11:52.373985+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"230'11298","reported_seq":11402,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:03.335317+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:12:03.335317+0000","last_peered":"2024-09-14T04:12:03.335317+0000","last_clean":"2024-09-14T04:12:03.335317+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:12:03.335317+0000","last_undegraded":"2024-09-14T04:12:03.335317+0000","last_fullsized":"2024-09-14T04:12:03.335317+0000","mapping_epoch":184,"log_start":"199'4259","ondisk_log_start":"199'4259","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11298,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":479,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448007+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:12:02.448007+0000","last_peered":"2024-09-14T04:12:02.448007+0000","last_clean":"2024-09-14T04:12:02.448007+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:12:02.448007+0000","last_undegraded":"2024-09-14T04:12:02.448007+0000","last_fullsized":"2024-09-14T04:12:02.448007+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":242,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313420+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:11:45.313420+0000","last_peered":"2024-09-14T04:11:45.313420+0000","last_clean":"2024-09-14T04:11:45.313420+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:11:45.313420+0000","last_undegraded":"2024-09-14T04:11:45.313420+0000","last_fullsized":"2024-09-14T04:11:45.313420+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":275,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.447933+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:12:02.447933+0000","last_peered":"2024-09-14T04:12:02.447933+0000","last_clean":"2024-09-14T04:12:02.447933+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:12:02.447933+0000","last_undegraded":"2024-09-14T04:12:02.447933+0000","last_fullsized":"2024-09-14T04:12:02.447933+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.452346+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:12:02.452346+0000","last_peered":"2024-09-14T04:12:02.452346+0000","last_clean":"2024-09-14T04:12:02.452346+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:12:02.452346+0000","last_undegraded":"2024-09-14T04:12:02.452346+0000","last_fullsized":"2024-09-14T04:12:02.452346+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"230'11546","reported_seq":14879,"reported_epoch":230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:02.574914+0000","last_change":"2024-09-14T04:10:35.403730+0000","last_active":"2024-09-14T04:12:02.574914+0000","last_peered":"2024-09-14T04:12:02.574914+0000","last_clean":"2024-09-14T04:09:49.197821+0000","last_became_active":"2024-09-14T04:09:51.414801+0000","last_became_peered":"2024-09-14T04:09:51.414801+0000","last_unstale":"2024-09-14T04:12:02.574914+0000","last_undegraded":"2024-09-14T04:10:35.401112+0000","last_fullsized":"2024-09-14T04:09:51.215769+0000","mapping_epoch":202,"log_start":"198'3200","ondisk_log_start":"198'3200","created":184,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8346,"log_dups_size":0,"ondisk_log_size":8346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189267968,"num_objects":2888,"num_object_clones":0,"num_object_copies":8664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11546,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":992},{"shards":"2,7","objects":-1404}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":461,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.383616+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:11:52.383616+0000","last_peered":"2024-09-14T04:11:52.383616+0000","last_clean":"2024-09-14T04:11:52.383616+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:11:52.383616+0000","last_undegraded":"2024-09-14T04:11:52.383616+0000","last_fullsized":"2024-09-14T04:11:52.383616+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":338,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451555+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:12:02.451555+0000","last_peered":"2024-09-14T04:12:02.451555+0000","last_clean":"2024-09-14T04:12:02.451555+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:12:02.451555+0000","last_undegraded":"2024-09-14T04:12:02.451555+0000","last_fullsized":"2024-09-14T04:12:02.451555+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":282,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.447832+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:12:02.447832+0000","last_peered":"2024-09-14T04:12:02.447832+0000","last_clean":"2024-09-14T04:12:02.447832+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:12:02.447832+0000","last_undegraded":"2024-09-14T04:12:02.447832+0000","last_fullsized":"2024-09-14T04:12:02.447832+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":350,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448560+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:12:02.448560+0000","last_peered":"2024-09-14T04:12:02.448560+0000","last_clean":"2024-09-14T04:12:02.448560+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:12:02.448560+0000","last_undegraded":"2024-09-14T04:12:02.448560+0000","last_fullsized":"2024-09-14T04:12:02.448560+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"229'11324","reported_seq":12603,"reported_epoch":229,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:01.192620+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:12:01.192620+0000","last_peered":"2024-09-14T04:12:01.192620+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:12:01.192620+0000","last_undegraded":"2024-09-14T04:12:01.192620+0000","last_fullsized":"2024-09-14T04:12:01.192620+0000","mapping_epoch":210,"log_start":"199'4800","ondisk_log_start":"199'4800","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187432960,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2860,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2860}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":467,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.309950+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:11:45.309950+0000","last_peered":"2024-09-14T04:11:45.309950+0000","last_clean":"2024-09-14T04:11:45.309950+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:11:45.309950+0000","last_undegraded":"2024-09-14T04:11:45.309950+0000","last_fullsized":"2024-09-14T04:11:45.309950+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":213,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.307099+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:11:45.307099+0000","last_peered":"2024-09-14T04:11:45.307099+0000","last_clean":"2024-09-14T04:11:45.307099+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:11:45.307099+0000","last_undegraded":"2024-09-14T04:11:45.307099+0000","last_fullsized":"2024-09-14T04:11:45.307099+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":232,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:50.746929+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:11:50.746929+0000","last_peered":"2024-09-14T04:11:50.746929+0000","last_clean":"2024-09-14T04:11:50.746929+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:11:50.746929+0000","last_undegraded":"2024-09-14T04:11:50.746929+0000","last_fullsized":"2024-09-14T04:11:50.746929+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":227,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306130+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:11:45.306130+0000","last_peered":"2024-09-14T04:11:45.306130+0000","last_clean":"2024-09-14T04:11:45.306130+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:11:45.306130+0000","last_undegraded":"2024-09-14T04:11:45.306130+0000","last_fullsized":"2024-09-14T04:11:45.306130+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"229'11607","reported_seq":11662,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:01.063458+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:12:01.063458+0000","last_peered":"2024-09-14T04:12:01.063458+0000","last_clean":"2024-09-14T04:12:01.063458+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:12:01.063458+0000","last_undegraded":"2024-09-14T04:12:01.063458+0000","last_fullsized":"2024-09-14T04:12:01.063458+0000","mapping_epoch":184,"log_start":"198'3700","ondisk_log_start":"198'3700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195624960,"num_objects":2985,"num_object_clones":0,"num_object_copies":8955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11607,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":441,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.318401+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:11:45.318401+0000","last_peered":"2024-09-14T04:11:45.318401+0000","last_clean":"2024-09-14T04:11:45.318401+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:11:45.318401+0000","last_undegraded":"2024-09-14T04:11:45.318401+0000","last_fullsized":"2024-09-14T04:11:45.318401+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.364038+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:11:52.364038+0000","last_peered":"2024-09-14T04:11:52.364038+0000","last_clean":"2024-09-14T04:11:52.364038+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:11:52.364038+0000","last_undegraded":"2024-09-14T04:11:52.364038+0000","last_fullsized":"2024-09-14T04:11:52.364038+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":350,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448205+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:12:02.448205+0000","last_peered":"2024-09-14T04:12:02.448205+0000","last_clean":"2024-09-14T04:12:02.448205+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:12:02.448205+0000","last_undegraded":"2024-09-14T04:12:02.448205+0000","last_fullsized":"2024-09-14T04:12:02.448205+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":299,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448532+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:12:02.448532+0000","last_peered":"2024-09-14T04:12:02.448532+0000","last_clean":"2024-09-14T04:12:02.448532+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:12:02.448532+0000","last_undegraded":"2024-09-14T04:12:02.448532+0000","last_fullsized":"2024-09-14T04:12:02.448532+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"229'11365","reported_seq":12557,"reported_epoch":229,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:01.177855+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:12:01.177855+0000","last_peered":"2024-09-14T04:12:01.177855+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:12:01.177855+0000","last_undegraded":"2024-09-14T04:12:01.177855+0000","last_fullsized":"2024-09-14T04:12:01.177855+0000","mapping_epoch":210,"log_start":"199'4800","ondisk_log_start":"199'4800","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192020504,"num_objects":2931,"num_object_clones":0,"num_object_copies":8793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2931,"num_objects_unfound":0,"num_objects_dirty":2931,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11365,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":2931}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":475,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310778+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:11:45.310778+0000","last_peered":"2024-09-14T04:11:45.310778+0000","last_clean":"2024-09-14T04:11:45.310778+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:11:45.310778+0000","last_undegraded":"2024-09-14T04:11:45.310778+0000","last_fullsized":"2024-09-14T04:11:45.310778+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":281,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448090+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:12:02.448090+0000","last_peered":"2024-09-14T04:12:02.448090+0000","last_clean":"2024-09-14T04:12:02.448090+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:12:02.448090+0000","last_undegraded":"2024-09-14T04:12:02.448090+0000","last_fullsized":"2024-09-14T04:12:02.448090+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":244,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312643+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:11:45.312643+0000","last_peered":"2024-09-14T04:11:45.312643+0000","last_clean":"2024-09-14T04:11:45.312643+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:11:45.312643+0000","last_undegraded":"2024-09-14T04:11:45.312643+0000","last_fullsized":"2024-09-14T04:11:45.312643+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.449419+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:12:02.449419+0000","last_peered":"2024-09-14T04:12:02.449419+0000","last_clean":"2024-09-14T04:12:02.449419+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:12:02.449419+0000","last_undegraded":"2024-09-14T04:12:02.449419+0000","last_fullsized":"2024-09-14T04:12:02.449419+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"229'11279","reported_seq":11334,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:00.663134+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:12:00.663134+0000","last_peered":"2024-09-14T04:12:00.663134+0000","last_clean":"2024-09-14T04:12:00.663134+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:12:00.663134+0000","last_undegraded":"2024-09-14T04:12:00.663134+0000","last_fullsized":"2024-09-14T04:12:00.663134+0000","mapping_epoch":184,"log_start":"199'3912","ondisk_log_start":"199'3912","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193396736,"num_objects":2951,"num_object_clones":0,"num_object_copies":8853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11279,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":433,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305865+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:11:45.305865+0000","last_peered":"2024-09-14T04:11:45.305865+0000","last_clean":"2024-09-14T04:11:45.305865+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:11:45.305865+0000","last_undegraded":"2024-09-14T04:11:45.305865+0000","last_fullsized":"2024-09-14T04:11:45.305865+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":162,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313250+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:11:45.313250+0000","last_peered":"2024-09-14T04:11:45.313250+0000","last_clean":"2024-09-14T04:11:45.313250+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:11:45.313250+0000","last_undegraded":"2024-09-14T04:11:45.313250+0000","last_fullsized":"2024-09-14T04:11:45.313250+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":282,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448393+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:12:02.448393+0000","last_peered":"2024-09-14T04:12:02.448393+0000","last_clean":"2024-09-14T04:12:02.448393+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:12:02.448393+0000","last_undegraded":"2024-09-14T04:12:02.448393+0000","last_fullsized":"2024-09-14T04:12:02.448393+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":177,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.313419+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:11:45.313419+0000","last_peered":"2024-09-14T04:11:45.313419+0000","last_clean":"2024-09-14T04:11:45.313419+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:11:45.313419+0000","last_undegraded":"2024-09-14T04:11:45.313419+0000","last_fullsized":"2024-09-14T04:11:45.313419+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"229'11385","reported_seq":11440,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:12:01.145694+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:12:01.145694+0000","last_peered":"2024-09-14T04:12:01.145694+0000","last_clean":"2024-09-14T04:12:01.145694+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:12:01.145694+0000","last_undegraded":"2024-09-14T04:12:01.145694+0000","last_fullsized":"2024-09-14T04:12:01.145694+0000","mapping_epoch":184,"log_start":"199'4800","ondisk_log_start":"199'4800","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11385,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":505,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389618+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:11:52.389618+0000","last_peered":"2024-09-14T04:11:52.389618+0000","last_clean":"2024-09-14T04:11:52.389618+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:11:52.389618+0000","last_undegraded":"2024-09-14T04:11:52.389618+0000","last_fullsized":"2024-09-14T04:11:52.389618+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":56,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451050+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:12:02.451050+0000","last_peered":"2024-09-14T04:12:02.451050+0000","last_clean":"2024-09-14T04:12:02.451050+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:12:02.451050+0000","last_undegraded":"2024-09-14T04:12:02.451050+0000","last_fullsized":"2024-09-14T04:12:02.451050+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":227,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.318651+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:11:45.318651+0000","last_peered":"2024-09-14T04:11:45.318651+0000","last_clean":"2024-09-14T04:11:45.318651+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:11:45.318651+0000","last_undegraded":"2024-09-14T04:11:45.318651+0000","last_fullsized":"2024-09-14T04:11:45.318651+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.738510+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:11:45.738510+0000","last_peered":"2024-09-14T04:11:45.738510+0000","last_clean":"2024-09-14T04:11:45.738510+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:11:45.738510+0000","last_undegraded":"2024-09-14T04:11:45.738510+0000","last_fullsized":"2024-09-14T04:11:45.738510+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305400+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:11:45.305400+0000","last_peered":"2024-09-14T04:11:45.305400+0000","last_clean":"2024-09-14T04:11:45.305400+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:11:45.305400+0000","last_undegraded":"2024-09-14T04:11:45.305400+0000","last_fullsized":"2024-09-14T04:11:45.305400+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306024+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:11:45.306024+0000","last_peered":"2024-09-14T04:11:45.306024+0000","last_clean":"2024-09-14T04:11:45.306024+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:11:45.306024+0000","last_undegraded":"2024-09-14T04:11:45.306024+0000","last_fullsized":"2024-09-14T04:11:45.306024+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":282,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448390+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:12:02.448390+0000","last_peered":"2024-09-14T04:12:02.448390+0000","last_clean":"2024-09-14T04:12:02.448390+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:12:02.448390+0000","last_undegraded":"2024-09-14T04:12:02.448390+0000","last_fullsized":"2024-09-14T04:12:02.448390+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":236,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312424+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:11:45.312424+0000","last_peered":"2024-09-14T04:11:45.312424+0000","last_clean":"2024-09-14T04:11:45.312424+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:11:45.312424+0000","last_undegraded":"2024-09-14T04:11:45.312424+0000","last_fullsized":"2024-09-14T04:11:45.312424+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":197,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306267+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:11:45.306267+0000","last_peered":"2024-09-14T04:11:45.306267+0000","last_clean":"2024-09-14T04:11:45.306267+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:11:45.306267+0000","last_undegraded":"2024-09-14T04:11:45.306267+0000","last_fullsized":"2024-09-14T04:11:45.306267+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":159,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.312149+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:11:45.312149+0000","last_peered":"2024-09-14T04:11:45.312149+0000","last_clean":"2024-09-14T04:11:45.312149+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:11:45.312149+0000","last_undegraded":"2024-09-14T04:11:45.312149+0000","last_fullsized":"2024-09-14T04:11:45.312149+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443502+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:12:02.443502+0000","last_peered":"2024-09-14T04:12:02.443502+0000","last_clean":"2024-09-14T04:12:02.443502+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:12:02.443502+0000","last_undegraded":"2024-09-14T04:12:02.443502+0000","last_fullsized":"2024-09-14T04:12:02.443502+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":158,"reported_epoch":229,"state":"active+clean","last_fresh":"2024-09-14T04:11:52.389462+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:11:52.389462+0000","last_peered":"2024-09-14T04:11:52.389462+0000","last_clean":"2024-09-14T04:11:52.389462+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:11:52.389462+0000","last_undegraded":"2024-09-14T04:11:52.389462+0000","last_fullsized":"2024-09-14T04:11:52.389462+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:46.150931+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:11:46.150931+0000","last_peered":"2024-09-14T04:11:46.150931+0000","last_clean":"2024-09-14T04:11:46.150931+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:11:46.150931+0000","last_undegraded":"2024-09-14T04:11:46.150931+0000","last_fullsized":"2024-09-14T04:11:46.150931+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445140+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:12:02.445140+0000","last_peered":"2024-09-14T04:12:02.445140+0000","last_clean":"2024-09-14T04:12:02.445140+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:12:02.445140+0000","last_undegraded":"2024-09-14T04:12:02.445140+0000","last_fullsized":"2024-09-14T04:12:02.445140+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450359+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:12:02.450359+0000","last_peered":"2024-09-14T04:12:02.450359+0000","last_clean":"2024-09-14T04:12:02.450359+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:12:02.450359+0000","last_undegraded":"2024-09-14T04:12:02.450359+0000","last_fullsized":"2024-09-14T04:12:02.450359+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.447881+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:12:02.447881+0000","last_peered":"2024-09-14T04:12:02.447881+0000","last_clean":"2024-09-14T04:12:02.447881+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:12:02.447881+0000","last_undegraded":"2024-09-14T04:12:02.447881+0000","last_fullsized":"2024-09-14T04:12:02.447881+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306502+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:11:45.306502+0000","last_peered":"2024-09-14T04:11:45.306502+0000","last_clean":"2024-09-14T04:11:45.306502+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:11:45.306502+0000","last_undegraded":"2024-09-14T04:11:45.306502+0000","last_fullsized":"2024-09-14T04:11:45.306502+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":206,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446717+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:12:02.446717+0000","last_peered":"2024-09-14T04:12:02.446717+0000","last_clean":"2024-09-14T04:12:02.446717+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:12:02.446717+0000","last_undegraded":"2024-09-14T04:12:02.446717+0000","last_fullsized":"2024-09-14T04:12:02.446717+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":325,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.448923+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:12:02.448923+0000","last_peered":"2024-09-14T04:12:02.448923+0000","last_clean":"2024-09-14T04:12:02.448923+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:12:02.448923+0000","last_undegraded":"2024-09-14T04:12:02.448923+0000","last_fullsized":"2024-09-14T04:12:02.448923+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443511+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:12:02.443511+0000","last_peered":"2024-09-14T04:12:02.443511+0000","last_clean":"2024-09-14T04:12:02.443511+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:12:02.443511+0000","last_undegraded":"2024-09-14T04:12:02.443511+0000","last_fullsized":"2024-09-14T04:12:02.443511+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.305227+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:11:45.305227+0000","last_peered":"2024-09-14T04:11:45.305227+0000","last_clean":"2024-09-14T04:11:45.305227+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:11:45.305227+0000","last_undegraded":"2024-09-14T04:11:45.305227+0000","last_fullsized":"2024-09-14T04:11:45.305227+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":235,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306314+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:11:45.306314+0000","last_peered":"2024-09-14T04:11:45.306314+0000","last_clean":"2024-09-14T04:11:45.306314+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:11:45.306314+0000","last_undegraded":"2024-09-14T04:11:45.306314+0000","last_fullsized":"2024-09-14T04:11:45.306314+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306910+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:11:45.306910+0000","last_peered":"2024-09-14T04:11:45.306910+0000","last_clean":"2024-09-14T04:11:45.306910+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:11:45.306910+0000","last_undegraded":"2024-09-14T04:11:45.306910+0000","last_fullsized":"2024-09-14T04:11:45.306910+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":222,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306639+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:11:45.306639+0000","last_peered":"2024-09-14T04:11:45.306639+0000","last_clean":"2024-09-14T04:11:45.306639+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:11:45.306639+0000","last_undegraded":"2024-09-14T04:11:45.306639+0000","last_fullsized":"2024-09-14T04:11:45.306639+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":286,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451474+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:12:02.451474+0000","last_peered":"2024-09-14T04:12:02.451474+0000","last_clean":"2024-09-14T04:12:02.451474+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:12:02.451474+0000","last_undegraded":"2024-09-14T04:12:02.451474+0000","last_fullsized":"2024-09-14T04:12:02.451474+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":104,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.310914+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:11:45.310914+0000","last_peered":"2024-09-14T04:11:45.310914+0000","last_clean":"2024-09-14T04:11:45.310914+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:11:45.310914+0000","last_undegraded":"2024-09-14T04:11:45.310914+0000","last_fullsized":"2024-09-14T04:11:45.310914+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":228,"state":"active+clean","last_fresh":"2024-09-14T04:11:45.306184+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:11:45.306184+0000","last_peered":"2024-09-14T04:11:45.306184+0000","last_clean":"2024-09-14T04:11:45.306184+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:11:45.306184+0000","last_undegraded":"2024-09-14T04:11:45.306184+0000","last_fullsized":"2024-09-14T04:11:45.306184+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":58,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443631+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:12:02.443631+0000","last_peered":"2024-09-14T04:12:02.443631+0000","last_clean":"2024-09-14T04:12:02.443631+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:12:02.443631+0000","last_undegraded":"2024-09-14T04:12:02.443631+0000","last_fullsized":"2024-09-14T04:12:02.443631+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3038707736,"num_objects":46368,"num_object_clones":0,"num_object_copies":139104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9950,"num_objects_misplaced":14122,"num_objects_unfound":0,"num_objects_dirty":46368,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183908,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10992,"num_bytes_recovered":694026240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9875894272,"data_stored":9875882056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119765,"ondisk_log_size":119765,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201657,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1666304,"kb_used_data":887032,"kb_used_omap":1,"kb_used_meta":779262,"kb_avail":92078848,"statfs":{"total":95995035648,"available":94288740352,"internally_reserved":0,"allocated":908320768,"data_stored":906870747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":797964747},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463295,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1977728,"kb_used_data":1328824,"kb_used_omap":1,"kb_used_meta":648830,"kb_avail":91767424,"statfs":{"total":95995035648,"available":93969842176,"internally_reserved":0,"allocated":1360715776,"data_stored":1359266258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":664402375},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724931,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787444,"kb_used_data":809644,"kb_used_omap":1,"kb_used_meta":977790,"kb_avail":91957708,"statfs":{"total":95995035648,"available":94164692992,"internally_reserved":0,"allocated":829075456,"data_stored":827633719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1001257414},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986567,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2695488,"kb_used_data":1884216,"kb_used_omap":1,"kb_used_meta":811262,"kb_avail":91049664,"statfs":{"total":95995035648,"available":93234855936,"internally_reserved":0,"allocated":1929437184,"data_stored":1927986878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":830732742},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748596,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588148,"kb_used_data":1197932,"kb_used_omap":4,"kb_used_meta":390203,"kb_avail":92157004,"statfs":{"total":95995035648,"available":94368772096,"internally_reserved":0,"allocated":1226682368,"data_stored":1225240367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":399568428},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912123,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875700,"kb_used_data":1634604,"kb_used_omap":6,"kb_used_meta":241017,"kb_avail":91869452,"statfs":{"total":95995035648,"available":94074318848,"internally_reserved":0,"allocated":1673834496,"data_stored":1672392997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":246801977},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468140,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214588,"kb_used_data":714800,"kb_used_omap":5,"kb_used_meta":499770,"kb_avail":92530564,"statfs":{"total":95995035648,"available":94751297536,"internally_reserved":0,"allocated":731955200,"data_stored":730515664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":511764831},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342255,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1460424,"kb_used_data":1280572,"kb_used_omap":10,"kb_used_meta":179829,"kb_avail":92284728,"statfs":{"total":95995035648,"available":94499561472,"internally_reserved":0,"allocated":1311305728,"data_stored":1309855943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":184145106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1308164096,"data_stored":1308164096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":725024768,"data_stored":725024768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1662844928,"data_stored":1662844928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1205211136,"data_stored":1205207064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1915293696,"data_stored":1915289624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":820903936,"data_stored":820903936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1338773504,"data_stored":1338769432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":899678208,"data_stored":899678208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:12:07.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:12:07.746 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:07.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:07.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:07.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:07.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:08.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:08.082 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:08.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:08.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:08.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:08.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:08.405 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:08 smithi001 ceph-mon[107399]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:08.406 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:08 smithi001 ceph-mon[107399]: pgmap v787: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9962/132153 objects degraded (7.538%); 13460/132153 objects misplaced (10.185%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:08.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:08 smithi001 ceph-mon[112165]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:08.406 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:08 smithi001 ceph-mon[112165]: pgmap v787: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9962/132153 objects degraded (7.538%); 13460/132153 objects misplaced (10.185%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:08.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:08.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:08 smithi012 ceph-mon[101180]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:08 smithi012 ceph-mon[101180]: pgmap v787: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9962/132153 objects degraded (7.538%); 13460/132153 objects misplaced (10.185%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:08.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:08.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:08.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:08.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:08.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:08.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:08.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:08.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:09.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.067 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:09.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:09.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:12:09.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:09.213 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:09.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9962/132153 objects degraded (7.538%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:09.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:09.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:09.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9962/132153 objects degraded (7.538%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9962/132153 objects degraded (7.538%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:09.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:09.521 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:12:09.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:09.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:09.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:09.776 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:09.878 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:09.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:09.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.187 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:10 smithi012 ceph-mon[101180]: pgmap v788: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 9962/129591 objects degraded (7.687%); 13460/129591 objects misplaced (10.387%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:10.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:10.435 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:10.442 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:12:10.478 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:10.506 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:10 smithi001 ceph-mon[107399]: pgmap v788: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 9962/129591 objects degraded (7.687%); 13460/129591 objects misplaced (10.387%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:10.506 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:10 smithi001 ceph-mon[112165]: pgmap v788: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 9962/129591 objects degraded (7.687%); 13460/129591 objects misplaced (10.387%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:10.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.700 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:10.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:10.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:10.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:10.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:11.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:11.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:11.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:11.262 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:11.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:11.392 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:11.494 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:11.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:11.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:11.631 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:11.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:11.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:11.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:11.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:11.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:12.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:12.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:12.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:12.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:12.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:12.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:12.462 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:12 smithi001 ceph-mon[107399]: pgmap v789: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9962/126180 objects degraded (7.895%); 13460/126180 objects misplaced (10.667%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:12.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:12 smithi001 ceph-mon[112165]: pgmap v789: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9962/126180 objects degraded (7.895%); 13460/126180 objects misplaced (10.667%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:12.486 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:12 smithi012 ceph-mon[101180]: pgmap v789: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9962/126180 objects degraded (7.895%); 13460/126180 objects misplaced (10.667%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:12.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:12.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:12.638 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:12.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:12.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:12.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:12.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:12.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:12.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:13.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:13.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:13.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:13.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:191875 10.c 10:33f8fa7a:::benchmark_data_smithi001_190881_object76735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.235169+0000", 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.051894218999999998, 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051927781999999999, 2024-09-14T04:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191875 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235169+0000", 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235169+0000", 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235167+0000", 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235178+0000", 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1073999999999999e-05 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235195+0000", 2024-09-14T04:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7176999999999998e-05 2024-09-14T04:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235204+0000", 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1850000000000004e-06 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235351+0000", 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147484 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.235657+0000", 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030645800000000001 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.236083+0000", 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042550699999999999 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.245462+0000", 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093785050000000005 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.245944+0000", 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048254800000000002 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:191904 10.c 10:326bd8e4:::benchmark_data_smithi001_190881_object76764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.273744+0000", 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.013318806000000001, 2024-09-14T04:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013443112, 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191904 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273744+0000", 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273744+0000", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273743+0000", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273749+0000", 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9129999999999997e-06 2024-09-14T04:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273767+0000", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7076999999999999e-05 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273775+0000", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3739999999999998e-06 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273827+0000", 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2027999999999997e-05 2024-09-14T04:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.273947+0000", 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012050200000000001 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.274057+0000", 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109368 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283212+0000", 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091552319999999993 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:191905 10.1 e230/184 10:8bdabce6:::benchmark_data_smithi001_190881_object76765:head v 230'12052, mlcod=230'12048)", 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.276700+0000", 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.010362777, 2024-09-14T04:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010562624, 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276700+0000", 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276700+0000", 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276699+0000", 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276707+0000", 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4669999999999998e-06 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276724+0000", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6997e-05 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276730+0000", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.815e-06 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.280822+0000", 2024-09-14T04:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040918320000000001 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.280851+0000", 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8926999999999999e-05 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.311 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:191907 10.c 10:32961738:::benchmark_data_smithi001_190881_object76767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:12:13.311 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.276753+0000", 2024-09-14T04:12:13.311 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.010309821, 2024-09-14T04:12:13.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010568442000000001, 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191907 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276753+0000", 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276753+0000", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276752+0000", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276756+0000", 2024-09-14T04:12:13.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6979999999999997e-06 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276768+0000", 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1926999999999999e-05 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.276774+0000", 2024-09-14T04:12:13.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9190000000000004e-06 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.278123+0000", 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013486730000000001 2024-09-14T04:12:13.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.278236+0000", 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001131 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.278354+0000", 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011817200000000001 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.317 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.318 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:191909 10.c 10:398bcbfb:::benchmark_data_smithi001_190881_object76769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:12:13.318 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.279977+0000", 2024-09-14T04:12:13.318 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0070861270000000002, 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074135859999999998, 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:13.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191909 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.279977+0000", 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.279977+0000", 2024-09-14T04:12:13.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.279975+0000", 2024-09-14T04:12:13.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.279982+0000", 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4069999999999998e-06 2024-09-14T04:12:13.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.280010+0000", 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8299000000000001e-05 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.280018+0000", 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8399999999999995e-06 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283235+0000", 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003217714 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283352+0000", 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116246 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283471+0000", 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011896900000000001 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.327 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:191912 10.c 10.79257fc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.283053+0000", 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0040094320000000003, 2024-09-14T04:12:13.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044045300000000003, 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191912 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283053+0000", 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283053+0000", 2024-09-14T04:12:13.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283052+0000", 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:13.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283061+0000", 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0620000000000007e-06 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283077+0000", 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6269999999999998e-05 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283087+0000", 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2e-06 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:191910 10.8 e230/184 10:1525a0b6:::benchmark_data_smithi001_190881_object76770:head v 230'11729, mlcod=230'11727)", 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.283561+0000", 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0035017469999999999, 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039548750000000001, 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.333 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283561+0000", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283561+0000", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283560+0000", 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283571+0000", 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1073000000000001e-05 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283581+0000", 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2189999999999993e-06 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283586+0000", 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7899999999999996e-06 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.284007+0000", 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042076400000000001 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.284057+0000", 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9429999999999999e-05 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:191875 10.c e230/184)", 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.283601+0000", 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0034620290000000001, 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039754830000000001, 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283601+0000", 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283601+0000", 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283601+0000", 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283603+0000", 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6989999999999999e-06 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283609+0000", 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2260000000000002e-06 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283614+0000", 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2699999999999998e-06 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:191904 10.c e230/184)", 2024-09-14T04:12:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.283782+0000", 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0032804879999999998, 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038356340000000001, 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283782+0000", 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283782+0000", 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283782+0000", 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283788+0000", 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8860000000000001e-06 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283800+0000", 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1793e-05 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283805+0000", 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.383e-06 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:191907 10.c e230/184)", 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:13.283816+0000", 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0032468470000000002, 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038489570000000001, 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283816+0000", 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283816+0000", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283816+0000", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283818+0000", 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4729999999999999e-06 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283823+0000", 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9579999999999998e-06 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:13.283827+0000", 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4960000000000001e-06 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:12:13.347 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 10 2024-09-14T04:12:13.348 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:12:13.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:13.383 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:13.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:13.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:13.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:13.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:13.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:13.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:13.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:13.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:13.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:14.055 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:14.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:14.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:14.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:12:14.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:14.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:14.259 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[107399]: pgmap v790: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9996/119901 objects degraded (8.337%); 12801/119901 objects misplaced (10.676%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9996/119901 objects degraded (8.337%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[112165]: pgmap v790: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9996/119901 objects degraded (8.337%); 12801/119901 objects misplaced (10.676%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9996/119901 objects degraded (8.337%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:14.291 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:12:14.346 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:14.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:14.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:14.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:14.518 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:12:14.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:14 smithi012 ceph-mon[101180]: pgmap v790: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 9996/119901 objects degraded (8.337%); 12801/119901 objects misplaced (10.676%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:12:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9996/119901 objects degraded (8.337%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:12:14.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:14.666 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:14.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:14.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:14.838 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:14.838 INFO:teuthology.orchestra.run.smithi001.stdout:{"fsid":"5626d550-724a-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9996/119901 objects degraded (8.337%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1156,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":231,"num_osds":8,"num_up_osds":8,"osd_up_since":1726286894,"num_in_osds":8,"osd_in_since":1726286805,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":93},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":39967,"data_bytes":2618103805,"bytes_used":13415014400,"bytes_avail":754545270784,"bytes_total":767960285184,"degraded_objects":9996,"degraded_total":119901,"degraded_ratio":0.08336877924287539,"misplaced_objects":12801,"misplaced_total":119901,"misplaced_ratio":0.10676307954062102,"recovering_objects_per_sec":38,"recovering_bytes_per_sec":2537053,"recovering_keys_per_sec":0,"num_objects_recovered":416,"num_bytes_recovered":27262976,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":727},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi001.ytjgbj","status":"up:active","gid":24557}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-14T04:09:15.106156+0000","services":{}},"progress_events":{}} 2024-09-14T04:12:14.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:14.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:14.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:15.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:15.087 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:15.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:15.220 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:15.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:15.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:15 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:15.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:12:15.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:15 smithi012 ceph-mon[101180]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T04:12:15.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2298499181' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:15.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:15.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:15.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[107399]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2298499181' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1260843550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:12:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[112165]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T04:12:15.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2298499181' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:15.499 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:15.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:15.642 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:15.744 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:15.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.782 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:15.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:15.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:15.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:16.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:16.035 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:12:16.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:12:16.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:16.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.225 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:16.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:16.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:16.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:16.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:16.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:16.547 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:16 smithi012 ceph-mon[101180]: pgmap v792: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9996/117918 objects degraded (8.477%); 12801/117918 objects misplaced (10.856%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:16.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:16.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:16.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:16 smithi001 ceph-mon[107399]: pgmap v792: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9996/117918 objects degraded (8.477%); 12801/117918 objects misplaced (10.856%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:16.706 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:16 smithi001 ceph-mon[112165]: pgmap v792: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9996/117918 objects degraded (8.477%); 12801/117918 objects misplaced (10.856%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:12:16.715 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:12:16.715 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:12:16.715 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:12:16.715 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:12:16.715 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:12:16.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:16.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:16.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:16.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:16.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:16] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-09-14T04:12:17.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:17.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:17.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:12:17.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:17.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:17.316 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:17.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:17.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:17.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:12:17.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:17.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:17.656 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:17.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:17.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:17.881 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:12:17.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:17.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:17.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:18.062 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:18.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:18.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:18.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:18.262 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:18.262 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:18.324 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:18 smithi012 ceph-mon[101180]: pgmap v793: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 9996/110589 objects degraded (9.039%); 12087/110589 objects misplaced (10.930%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:18.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:18.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:18.420 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:18 smithi001 ceph-mon[107399]: pgmap v793: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 9996/110589 objects degraded (9.039%); 12087/110589 objects misplaced (10.930%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:18.422 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:18 smithi001 ceph-mon[112165]: pgmap v793: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 9996/110589 objects degraded (9.039%); 12087/110589 objects misplaced (10.930%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:18.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:18.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:18.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:18.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:18.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:18.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:18.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:18.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:18.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:18.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:19.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:19.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:19.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:19.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:19.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:19.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:19.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:19.466 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9996/110589 objects degraded (9.039%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:19.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:19.518 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:19.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9996/110589 objects degraded (9.039%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:19.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9996/110589 objects degraded (9.039%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:19.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:19.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:19.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:19.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:19.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:19.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:19.933 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:12:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:162977 9.3 9:c2d413a3:::benchmark_data_smithi001_156540_object31383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 354.72953901, 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14203296300000001, 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 162977 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200126+0000", 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200124+0000", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200130+0000", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0850000000000002e-06 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200144+0000", 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3104e-05 2024-09-14T04:12:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.200151+0000", 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2250000000000003e-06 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201100+0000", 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094949499999999996 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201218+0000", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117963 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.201348+0000", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012961299999999999 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206267+0000", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049196209999999999 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206296+0000", 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8626e-05 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212462+0000", 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006165472 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.341995+0000", 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12953315100000001 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.342049+0000", 2024-09-14T04:12:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3891999999999998e-05 2024-09-14T04:12:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.342159+0000", 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110272 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:162980 9.3 9:cb003418:::benchmark_data_smithi001_156540_object31386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 354.724813031, 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14076661400000001, 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 162980 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204852+0000", 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204850+0000", 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204856+0000", 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.84e-06 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204867+0000", 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0231000000000001e-05 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.204873+0000", 2024-09-14T04:12:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7889999999999997e-06 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206317+0000", 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014438579999999999 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206431+0000", 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011421000000000001 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.206554+0000", 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012220300000000001 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212466+0000", 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059127829999999996 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.212489+0000", 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2138999999999999e-05 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.213175+0000", 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00068672299999999998 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.344992+0000", 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13181647899999999 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.345505+0000", 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051285999999999997 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:25.345618+0000", 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011390100000000001 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166779 9.d 9:b228385c:::benchmark_data_smithi001_156540_object35185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:12:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:12:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.73863248499998, 2024-09-14T04:12:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19259300700000001, 2024-09-14T04:12:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166779 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191032+0000", 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191031+0000", 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191036+0000", 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6450000000000004e-06 2024-09-14T04:12:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191046+0000", 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6949999999999995e-06 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191052+0000", 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2009999999999996e-06 2024-09-14T04:12:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191095+0000", 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3046000000000001e-05 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191200+0000", 2024-09-14T04:12:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105394 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.191304+0000", 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104038 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.197245+0000", 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059409659999999998 2024-09-14T04:12:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.208912+0000", 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011666434999999999 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383544+0000", 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17463272099999999 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383590+0000", 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5046999999999997e-05 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383625+0000", 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5918999999999999e-05 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166788 9.d 9:b0a806cb:::benchmark_data_smithi001_156540_object35194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.729382585, 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18345835699999999, 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166788 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200282+0000", 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200287+0000", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0119999999999996e-06 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200301+0000", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4596999999999999e-05 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.200308+0000", 2024-09-14T04:12:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7349999999999999e-06 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201962+0000", 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001654483 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.203690+0000", 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001727803 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.203796+0000", 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105812 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.210818+0000", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070218889999999999 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.210946+0000", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012832300000000001 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383661+0000", 2024-09-14T04:12:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17271468100000001 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383708+0000", 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7357000000000001e-05 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383741+0000", 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2447999999999998e-05 2024-09-14T04:12:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166802 9.d 9:be97eccd:::benchmark_data_smithi001_156540_object35208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.712701837, 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.166921759, 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166802 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216963+0000", 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216965+0000", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5220000000000002e-06 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216971+0000", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7679999999999997e-06 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.216975+0000", 2024-09-14T04:12:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4889999999999997e-06 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217020+0000", 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4070000000000003e-05 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217127+0000", 2024-09-14T04:12:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010763400000000001 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.217236+0000", 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108507 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.229705+0000", 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012468918000000001 2024-09-14T04:12:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.234522+0000", 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048169500000000004 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383773+0000", 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14925143099999999 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383850+0000", 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6933000000000004e-05 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383885+0000", 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4947999999999998e-05 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166799 9.1 e153/149 9:8c8a635f:::benchmark_data_smithi001_156540_object35205:head v 153'10484, mlcod=153'10481)", 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.70794248499999, 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.159769142, 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221722+0000", 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221728+0000", 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1840000000000004e-06 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221738+0000", 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.027e-05 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.221745+0000", 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2920000000000001e-06 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.226758+0000", 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050130519999999996 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.226786+0000", 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8493999999999999e-05 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381450+0000", 2024-09-14T04:12:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15466439100000001 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381492+0000", 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1084000000000002e-05 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166808 9.1 e153/149 9:85c01d48:::benchmark_data_smithi001_156540_object35214:head v 153'10485, mlcod=153'10481)", 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.69745400199997, 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.149302823, 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232211+0000", 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232210+0000", 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232217+0000", 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2280000000000001e-06 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232235+0000", 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7819e-05 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232242+0000", 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6089999999999997e-06 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232340+0000", 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8549999999999997e-05 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232361+0000", 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0945999999999999e-05 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381494+0000", 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14913268199999999 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381514+0000", 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9678999999999999e-05 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166805 9.7 e153/149 9:e1bea35c:::benchmark_data_smithi001_156540_object35211:head v 153'10387, mlcod=153'10385)", 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.696690913, 2024-09-14T04:12:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14852269200000001, 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232974+0000", 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232977+0000", 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.551e-06 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232983+0000", 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9780000000000002e-06 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.232987+0000", 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.997e-06 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233201+0000", 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021345899999999999 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.233225+0000", 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4219e-05 2024-09-14T04:12:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381381+0000", 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14815647600000001 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381497+0000", 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115259 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166809 9.2 e153/149 9:4e12f0fd:::benchmark_data_smithi001_156540_object35215:head v 153'10294, mlcod=153'10291)", 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 349.69371075800001, 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15257738200000001, 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235954+0000", 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235953+0000", 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235960+0000", 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5910000000000001e-06 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235971+0000", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0769e-05 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.235977+0000", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2639999999999997e-06 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.236047+0000", 2024-09-14T04:12:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9840999999999998e-05 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.381832+0000", 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14578494 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.388490+0000", 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066577950000000002 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.388532+0000", 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2002000000000002e-05 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178529 9.3 9:c63b081d:::benchmark_data_smithi001_156540_object46935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:12:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 334.59564673, 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1003067309999999, 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178529 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334018+0000", 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334017+0000", 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334021+0000", 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0679999999999998e-06 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334030+0000", 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4860000000000003e-06 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334037+0000", 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.939e-06 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334160+0000", 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012251300000000001 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334282+0000", 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012243599999999999 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334405+0000", 2024-09-14T04:12:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012238400000000001 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356132+0000", 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021727265999999999 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356140+0000", 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4810000000000006e-06 2024-09-14T04:12:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332059+0000", 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97591887700000002 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332074+0000", 2024-09-14T04:12:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5068999999999999e-05 2024-09-14T04:12:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404414+0000", 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.072339559 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404564+0000", 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014992900000000001 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.434245+0000", 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029680840999999999 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.434325+0000", 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0191000000000005e-05 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178530 9.3 9:ce32ae9f:::benchmark_data_smithi001_156540_object46936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 334.59562093099998, 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1012480839999998, 2024-09-14T04:12:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178530 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:12:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334044+0000", 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:12:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334046+0000", 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1660000000000001e-06 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334051+0000", 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4229999999999996e-06 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.334056+0000", 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6410000000000001e-06 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340669+0000", 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066129869999999999 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340821+0000", 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152435 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3,5", 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.340975+0000", 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015369299999999999 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356177+0000", 2024-09-14T04:12:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015202126999999999 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.356184+0000", 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4230000000000002e-06 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332331+0000", 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.976147133 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.332341+0000", 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0288999999999999e-05 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404844+0000", 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0725033310000001 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.404952+0000", 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108185 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.435228+0000", 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030275077000000001 2024-09-14T04:12:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.435292+0000", 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4493999999999996e-05 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178555 9.3 9:c397c393:::benchmark_data_smithi001_156540_object46961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 333.500893082, 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0051882679999999, 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178555 2024-09-14T04:12:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428772+0000", 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428770+0000", 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428777+0000", 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1589999999999998e-06 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428789+0000", 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2266000000000001e-05 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428795+0000", 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9399999999999999e-06 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428845+0000", 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9852999999999999e-05 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.428854+0000", 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6830000000000003e-06 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405044+0000", 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97618959400000005 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405148+0000", 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010432799999999999 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:12:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.405258+0000", 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010979 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.416784+0000", 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011525933 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.422336+0000", 2024-09-14T04:12:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055522949999999996 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433810+0000", 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011473524000000001 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433862+0000", 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2414000000000003e-05 2024-09-14T04:12:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:47.433960+0000", 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7918000000000007e-05 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:180040 9.d 9:b3542a9b:::benchmark_data_smithi001_156540_object48446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:12:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 331.11623477400002, 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.138239108, 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180040 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813430+0000", 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813429+0000", 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813436+0000", 2024-09-14T04:12:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0389999999999999e-06 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813449+0000", 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3185e-05 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813457+0000", 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4100000000000002e-06 2024-09-14T04:12:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813503+0000", 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6489999999999997e-05 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813583+0000", 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0097000000000002e-05 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.813689+0000", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105437 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.817177+0000", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034882419999999999 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.817259+0000", 2024-09-14T04:12:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1843999999999998e-05 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951582+0000", 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.134323266 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951632+0000", 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0160000000000001e-05 2024-09-14T04:12:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.951669+0000", 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7150000000000002e-05 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:180046 9.d 9:ba9a6653:::benchmark_data_smithi001_156540_object48452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 331.11153006900003, 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.136132161, 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180046 2024-09-14T04:12:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818135+0000", 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818134+0000", 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818139+0000", 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9409999999999998e-06 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818147+0000", 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0180000000000001e-06 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818151+0000", 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5700000000000003e-06 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818184+0000", 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2175999999999998e-05 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818282+0000", 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8043999999999998e-05 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.818348+0000", 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6302000000000003e-05 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.821041+0000", 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026934300000000001 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.821810+0000", 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076834799999999997 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954149+0000", 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13233890200000001 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954232+0000", 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3170999999999994e-05 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:48.954267+0000", 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5178000000000001e-05 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:220052 9.3 9:c193233b:::benchmark_data_smithi001_156540_object88458:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 289.92757550099998, 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.131223174, 2024-09-14T04:12:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 220052 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:12:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002089+0000", 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002088+0000", 2024-09-14T04:12:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002094+0000", 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5979999999999996e-06 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002105+0000", 2024-09-14T04:12:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0869e-05 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.002111+0000", 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3219999999999998e-06 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005294+0000", 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003182706 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005457+0000", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000163201 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.005572+0000", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115293 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.009988+0000", 2024-09-14T04:12:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044159469999999999 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.010128+0000", 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014000300000000001 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133227+0000", 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12309882 2024-09-14T04:12:20.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133275+0000", 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7772000000000003e-05 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:07:30.133313+0000", 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7889999999999998e-05 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.052 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:161512 10.c 10:30e99e23:::benchmark_data_smithi001_190881_object46372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.087894+0000", 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.841770574999998, 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17573211899999999, 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:20.053 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 161512 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087894+0000", 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087894+0000", 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087892+0000", 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087901+0000", 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2810000000000008e-06 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087915+0000", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4423000000000001e-05 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.087923+0000", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1159999999999999e-06 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.088400+0000", 2024-09-14T04:12:20.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047739599999999999 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.088518+0000", 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117329 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.088635+0000", 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117534 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.101477+0000", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01284157 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.101899+0000", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042243899999999999 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263542+0000", 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16164262300000001 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263587+0000", 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4968e-05 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263626+0000", 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9517999999999997e-05 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:161513 10.c 10:358a40bb:::benchmark_data_smithi001_190881_object46373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.089737+0000", 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.839928266000001, 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17406521799999999, 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:20.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 161513 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089737+0000", 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089737+0000", 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089735+0000", 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089743+0000", 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0779999999999996e-06 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089755+0000", 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1576999999999999e-05 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.089762+0000", 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.294e-06 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.092820+0000", 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030579330000000001 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.092935+0000", 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115225 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.093048+0000", 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112241 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.101484+0000", 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084368459999999996 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.101992+0000", 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050814299999999996 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263664+0000", 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.161671972 2024-09-14T04:12:20.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263757+0000", 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2831000000000006e-05 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263802+0000", 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4564000000000002e-05 2024-09-14T04:12:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:161517 10.c 10:3cbbec78:::benchmark_data_smithi001_190881_object46377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.091134+0000", 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.83853122, 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.172961962, 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:12:20.068 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 161517 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091134+0000", 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091134+0000", 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091132+0000", 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091138+0000", 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8119999999999996e-06 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091150+0000", 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1985e-05 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.091157+0000", 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6590000000000001e-06 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.097036+0000", 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058792940000000002 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.097145+0000", 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010895000000000001 2024-09-14T04:12:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.097280+0000", 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013474799999999999 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.101489+0000", 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042091020000000002 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263841+0000", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16235271800000001 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263993+0000", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015163000000000001 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.264062+0000", 2024-09-14T04:12:20.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8727999999999997e-05 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.264096+0000", 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3818e-05 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:161525 10.1 e225/184 10:8b2d5aa2:::benchmark_data_smithi001_190881_object46385:head v 225'10154, mlcod=225'10152)", 2024-09-14T04:12:20.075 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.826520960000003, 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.190663473, 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103143+0000", 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103151+0000", 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7330000000000003e-06 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103162+0000", 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0943e-05 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103169+0000", 2024-09-14T04:12:20.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9859999999999997e-06 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263421+0000", 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16025208599999999 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263445+0000", 2024-09-14T04:12:20.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3892000000000001e-05 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293726+0000", 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030281499 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293807+0000", 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1364000000000002e-05 2024-09-14T04:12:20.080 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45035.0:161517 10.c e225/184 ondisk, result = 0)", 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.109758+0000", 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.819907110999999, 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15434213699999999, 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:12:20.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109758+0000", 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109758+0000", 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:12:20.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109756+0000", 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109766+0000", 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0012e-05 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263743+0000", 2024-09-14T04:12:20.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15397661700000001 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263751+0000", 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6820000000000003e-06 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263981+0000", 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022997600000000001 2024-09-14T04:12:20.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263988+0000", 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4669999999999998e-06 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.264100+0000", 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011171399999999999 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.085 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:12:20.086 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:12:20.086 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:12:20.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:20.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:20.187 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:20.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:20.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.361 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:20.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:20.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:20.515 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:20.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:20.577 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:20 smithi012 ceph-mon[101180]: pgmap v794: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 9996/107724 objects degraded (9.279%); 12087/107724 objects misplaced (11.220%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:20.577 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:20.607 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:20 smithi001 ceph-mon[112165]: pgmap v794: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 9996/107724 objects degraded (9.279%); 12087/107724 objects misplaced (11.220%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:20.607 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:20.607 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:20 smithi001 ceph-mon[107399]: pgmap v794: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 9996/107724 objects degraded (9.279%); 12087/107724 objects misplaced (11.220%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:20.607 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:20.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:20.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:20.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:20.792 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:20.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:20.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:20.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:21.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:21.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:21.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:12:21.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:21.183 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:12:21.184 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:12:21.224 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:12:21.224 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:12:21.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:21.262 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:12:21.263 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:12:21.296 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:12:21.296 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:12:21.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:21.339 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:12:21.339 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:12:21.369 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:12:21.370 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:12:21.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:21.409 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:12:21.409 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:12:21.434 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:21 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:21.434 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3131255519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:21.434 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:21 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:21.434 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3131255519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:21.435 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:12:21.435 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:12:21.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:21.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:21 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3131255519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:21.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:21.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:21.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:21.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:21.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:21.960 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:21.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:22.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:22.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:22.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:22.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:22.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:22.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:22.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:22 smithi001 ceph-mon[107399]: pgmap v795: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 9996/104091 objects degraded (9.603%); 12087/104091 objects misplaced (11.612%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:22.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:22 smithi001 ceph-mon[112165]: pgmap v795: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 9996/104091 objects degraded (9.603%); 12087/104091 objects misplaced (11.612%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:22.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:22.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:22 smithi012 ceph-mon[101180]: pgmap v795: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 9996/104091 objects degraded (9.603%); 12087/104091 objects misplaced (11.612%); 2.7 MiB/s, 42 objects/s recovering 2024-09-14T04:12:22.635 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:22.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:22.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:22.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:22.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:22.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:22.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:23.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:23.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:23.325 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:12:23.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:23.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:23.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:23.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:23.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:23 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:23.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:23.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:23 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:23.686 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:23.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:23 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:23.690 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:12:23.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:23.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:23.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:23.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:23.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:23.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:24.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:24.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:24.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:24.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:24.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:24.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:24.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:24.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.533 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:24.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:24 smithi012 ceph-mon[101180]: pgmap v796: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/96765 objects degraded (10.330%); 11347/96765 objects misplaced (11.726%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:24 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:12:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:24 smithi012 ceph-mon[101180]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T04:12:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9996/96765 objects degraded (10.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:24.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:24.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[107399]: pgmap v796: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/96765 objects degraded (10.330%); 11347/96765 objects misplaced (11.726%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[107399]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9996/96765 objects degraded (10.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[112165]: pgmap v796: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/96765 objects degraded (10.330%); 11347/96765 objects misplaced (11.726%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4120562197' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:24.691 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[112165]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T04:12:24.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9996/96765 objects degraded (10.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:24.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:24.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:24.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:24.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:24.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:25.025 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:25.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:25.055 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":797,"stamp":"2024-09-14T04:12:23.321841+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2112690173,"num_objects":32255,"num_object_clones":0,"num_object_copies":96765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9996,"num_objects_misplaced":11347,"num_objects_unfound":0,"num_objects_dirty":32255,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198111,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11932,"num_bytes_recovered":754058706,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121602,"ondisk_log_size":121602,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11736696,"kb_used_data":7136248,"kb_used_omap":33,"kb_used_meta":4600350,"kb_avail":738224520,"statfs":{"total":767960285184,"available":755941908480,"internally_reserved":0,"allocated":7307517952,"data_stored":7295844853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4710758836},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,7,4,6,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-490405888,"num_objects":-7483,"num_object_clones":0,"num_object_copies":-22449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1454,"num_objects_unfound":0,"num_objects_dirty":-7483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7483,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.061642"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":252,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.270558+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:12:14.270558+0000","last_peered":"2024-09-14T04:12:14.270558+0000","last_clean":"2024-09-14T04:12:14.270558+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:12:14.270558+0000","last_undegraded":"2024-09-14T04:12:14.270558+0000","last_fullsized":"2024-09-14T04:12:14.270558+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":204,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269911+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:12:14.269911+0000","last_peered":"2024-09-14T04:12:14.269911+0000","last_clean":"2024-09-14T04:12:14.269911+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:12:14.269911+0000","last_undegraded":"2024-09-14T04:12:14.269911+0000","last_fullsized":"2024-09-14T04:12:14.269911+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":41,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276593+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:12:14.276593+0000","last_peered":"2024-09-14T04:12:14.276593+0000","last_clean":"2024-09-14T04:12:14.276593+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:12:14.276593+0000","last_undegraded":"2024-09-14T04:12:14.276593+0000","last_fullsized":"2024-09-14T04:12:14.276593+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.268814+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:12:14.268814+0000","last_peered":"2024-09-14T04:12:14.268814+0000","last_clean":"2024-09-14T04:12:14.268814+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:12:14.268814+0000","last_undegraded":"2024-09-14T04:12:14.268814+0000","last_fullsized":"2024-09-14T04:12:14.268814+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269296+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:12:14.269296+0000","last_peered":"2024-09-14T04:12:14.269296+0000","last_clean":"2024-09-14T04:12:14.269296+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:12:14.269296+0000","last_undegraded":"2024-09-14T04:12:14.269296+0000","last_fullsized":"2024-09-14T04:12:14.269296+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":184,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.454443+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:12:02.454443+0000","last_peered":"2024-09-14T04:12:02.454443+0000","last_clean":"2024-09-14T04:12:02.454443+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:12:02.454443+0000","last_undegraded":"2024-09-14T04:12:02.454443+0000","last_fullsized":"2024-09-14T04:12:02.454443+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":162,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.279551+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:12:14.279551+0000","last_peered":"2024-09-14T04:12:14.279551+0000","last_clean":"2024-09-14T04:12:14.279551+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:12:14.279551+0000","last_undegraded":"2024-09-14T04:12:14.279551+0000","last_fullsized":"2024-09-14T04:12:14.279551+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":281,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.442662+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:12:02.442662+0000","last_peered":"2024-09-14T04:12:02.442662+0000","last_clean":"2024-09-14T04:12:02.442662+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:12:02.442662+0000","last_undegraded":"2024-09-14T04:12:02.442662+0000","last_fullsized":"2024-09-14T04:12:02.442662+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"231'12760","reported_seq":12817,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:21.810295+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:12:21.810295+0000","last_peered":"2024-09-14T04:12:21.810295+0000","last_clean":"2024-09-14T04:12:21.810295+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:12:21.810295+0000","last_undegraded":"2024-09-14T04:12:21.810295+0000","last_fullsized":"2024-09-14T04:12:21.810295+0000","mapping_epoch":184,"log_start":"198'3000","ondisk_log_start":"198'3000","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9760,"log_dups_size":0,"ondisk_log_size":9760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12760,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":242,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451388+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:12:02.451388+0000","last_peered":"2024-09-14T04:12:02.451388+0000","last_clean":"2024-09-14T04:12:02.451388+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:12:02.451388+0000","last_undegraded":"2024-09-14T04:12:02.451388+0000","last_fullsized":"2024-09-14T04:12:02.451388+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438470+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:12:02.438470+0000","last_peered":"2024-09-14T04:12:02.438470+0000","last_clean":"2024-09-14T04:12:02.438470+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:12:02.438470+0000","last_undegraded":"2024-09-14T04:12:02.438470+0000","last_fullsized":"2024-09-14T04:12:02.438470+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"231'12566","reported_seq":20732,"reported_epoch":231,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:21.294639+0000","last_change":"2024-09-14T04:10:35.400317+0000","last_active":"2024-09-14T04:12:21.294639+0000","last_peered":"2024-09-14T04:12:21.294639+0000","last_clean":"2024-09-14T04:09:49.203876+0000","last_became_active":"2024-09-14T04:09:51.437466+0000","last_became_peered":"2024-09-14T04:09:51.437466+0000","last_unstale":"2024-09-14T04:12:21.294639+0000","last_undegraded":"2024-09-14T04:09:51.253655+0000","last_fullsized":"2024-09-14T04:09:51.216149+0000","mapping_epoch":202,"log_start":"206'6000","ondisk_log_start":"206'6000","created":184,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6566,"log_dups_size":0,"ondisk_log_size":6566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:19:14.518927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131465216,"num_objects":2006,"num_object_clones":0,"num_object_copies":6018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1867,"num_objects_misplaced":113,"num_objects_unfound":0,"num_objects_dirty":2006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12566,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4042,"num_bytes_recovered":264896512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":113},{"shards":"3,6","objects":26}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":241,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451058+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:12:02.451058+0000","last_peered":"2024-09-14T04:12:02.451058+0000","last_clean":"2024-09-14T04:12:02.451058+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:12:02.451058+0000","last_undegraded":"2024-09-14T04:12:02.451058+0000","last_fullsized":"2024-09-14T04:12:02.451058+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446489+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:12:02.446489+0000","last_peered":"2024-09-14T04:12:02.446489+0000","last_clean":"2024-09-14T04:12:02.446489+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:12:02.446489+0000","last_undegraded":"2024-09-14T04:12:02.446489+0000","last_fullsized":"2024-09-14T04:12:02.446489+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"231'12432","reported_seq":13725,"reported_epoch":231,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:12:19.911257+0000","last_change":"2024-09-14T04:10:36.624348+0000","last_active":"2024-09-14T04:12:19.911257+0000","last_peered":"2024-09-14T04:12:19.911257+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:12:19.911257+0000","last_undegraded":"2024-09-14T04:12:19.911257+0000","last_fullsized":"2024-09-14T04:12:19.911257+0000","mapping_epoch":210,"log_start":"199'4500","ondisk_log_start":"199'4500","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7932,"log_dups_size":0,"ondisk_log_size":7932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138280960,"num_objects":2110,"num_object_clones":0,"num_object_copies":6330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7270,"num_objects_unfound":0,"num_objects_dirty":2110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12432,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":594,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-4753}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":212,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451490+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:12:02.451490+0000","last_peered":"2024-09-14T04:12:02.451490+0000","last_clean":"2024-09-14T04:12:02.451490+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:12:02.451490+0000","last_undegraded":"2024-09-14T04:12:02.451490+0000","last_fullsized":"2024-09-14T04:12:02.451490+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443640+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:12:02.443640+0000","last_peered":"2024-09-14T04:12:02.443640+0000","last_clean":"2024-09-14T04:12:02.443640+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:12:02.443640+0000","last_undegraded":"2024-09-14T04:12:02.443640+0000","last_fullsized":"2024-09-14T04:12:02.443640+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"231'12561","reported_seq":12667,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:22.639316+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:12:22.639316+0000","last_peered":"2024-09-14T04:12:22.639316+0000","last_clean":"2024-09-14T04:12:22.639316+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:12:22.639316+0000","last_undegraded":"2024-09-14T04:12:22.639316+0000","last_fullsized":"2024-09-14T04:12:22.639316+0000","mapping_epoch":184,"log_start":"199'4200","ondisk_log_start":"199'4200","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8361,"log_dups_size":0,"ondisk_log_size":8361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12561,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":218,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438847+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:12:02.438847+0000","last_peered":"2024-09-14T04:12:02.438847+0000","last_clean":"2024-09-14T04:12:02.438847+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:12:02.438847+0000","last_undegraded":"2024-09-14T04:12:02.438847+0000","last_fullsized":"2024-09-14T04:12:02.438847+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.270009+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:12:14.270009+0000","last_peered":"2024-09-14T04:12:14.270009+0000","last_clean":"2024-09-14T04:12:14.270009+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:12:14.270009+0000","last_undegraded":"2024-09-14T04:12:14.270009+0000","last_fullsized":"2024-09-14T04:12:14.270009+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"231'12151","reported_seq":19026,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:19.903121+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:12:19.903121+0000","last_peered":"2024-09-14T04:12:19.903121+0000","last_clean":"2024-09-14T04:12:19.903121+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:12:19.903121+0000","last_undegraded":"2024-09-14T04:12:19.903121+0000","last_fullsized":"2024-09-14T04:12:19.903121+0000","mapping_epoch":217,"log_start":"199'4200","ondisk_log_start":"199'4200","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133103616,"num_objects":2031,"num_object_clones":0,"num_object_copies":6093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12151,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":583,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:11.130622+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:12:11.130622+0000","last_peered":"2024-09-14T04:12:11.130622+0000","last_clean":"2024-09-14T04:12:11.130622+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:12:11.130622+0000","last_undegraded":"2024-09-14T04:12:11.130622+0000","last_fullsized":"2024-09-14T04:12:11.130622+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":58,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269541+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:12:14.269541+0000","last_peered":"2024-09-14T04:12:14.269541+0000","last_clean":"2024-09-14T04:12:14.269541+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:12:14.269541+0000","last_undegraded":"2024-09-14T04:12:14.269541+0000","last_fullsized":"2024-09-14T04:12:14.269541+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":217,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446367+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:12:02.446367+0000","last_peered":"2024-09-14T04:12:02.446367+0000","last_clean":"2024-09-14T04:12:02.446367+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:12:02.446367+0000","last_undegraded":"2024-09-14T04:12:02.446367+0000","last_fullsized":"2024-09-14T04:12:02.446367+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"231'12385","reported_seq":12491,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:22.615241+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:12:22.615241+0000","last_peered":"2024-09-14T04:12:22.615241+0000","last_clean":"2024-09-14T04:12:22.615241+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:12:22.615241+0000","last_undegraded":"2024-09-14T04:12:22.615241+0000","last_fullsized":"2024-09-14T04:12:22.615241+0000","mapping_epoch":184,"log_start":"199'4000","ondisk_log_start":"199'4000","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8385,"log_dups_size":0,"ondisk_log_size":8385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127598592,"num_objects":1947,"num_object_clones":0,"num_object_copies":5841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12385,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":452,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:06.775871+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:12:06.775871+0000","last_peered":"2024-09-14T04:12:06.775871+0000","last_clean":"2024-09-14T04:12:06.775871+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:12:06.775871+0000","last_undegraded":"2024-09-14T04:12:06.775871+0000","last_fullsized":"2024-09-14T04:12:06.775871+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":233,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443821+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:12:02.443821+0000","last_peered":"2024-09-14T04:12:02.443821+0000","last_clean":"2024-09-14T04:12:02.443821+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:12:02.443821+0000","last_undegraded":"2024-09-14T04:12:02.443821+0000","last_fullsized":"2024-09-14T04:12:02.443821+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445459+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:12:02.445459+0000","last_peered":"2024-09-14T04:12:02.445459+0000","last_clean":"2024-09-14T04:12:02.445459+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:12:02.445459+0000","last_undegraded":"2024-09-14T04:12:02.445459+0000","last_fullsized":"2024-09-14T04:12:02.445459+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"231'12290","reported_seq":14599,"reported_epoch":231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:21.272554+0000","last_change":"2024-09-14T04:11:13.382242+0000","last_active":"2024-09-14T04:12:21.272554+0000","last_peered":"2024-09-14T04:12:21.272554+0000","last_clean":"2024-09-14T04:09:49.209154+0000","last_became_active":"2024-09-14T04:11:13.358760+0000","last_became_peered":"2024-09-14T04:11:13.358760+0000","last_unstale":"2024-09-14T04:12:21.272554+0000","last_undegraded":"2024-09-14T04:11:13.259138+0000","last_fullsized":"2024-09-14T04:11:13.243084+0000","mapping_epoch":220,"log_start":"205'5700","ondisk_log_start":"205'5700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6590,"log_dups_size":0,"ondisk_log_size":6590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:46.130184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129892352,"num_objects":1982,"num_object_clones":0,"num_object_copies":5946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3837,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12290,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":73203712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":124},{"shards":"1,6,7","objects":1858}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446227+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:12:02.446227+0000","last_peered":"2024-09-14T04:12:02.446227+0000","last_clean":"2024-09-14T04:12:02.446227+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:12:02.446227+0000","last_undegraded":"2024-09-14T04:12:02.446227+0000","last_fullsized":"2024-09-14T04:12:02.446227+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.447921+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:12:02.447921+0000","last_peered":"2024-09-14T04:12:02.447921+0000","last_clean":"2024-09-14T04:12:02.447921+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:12:02.447921+0000","last_undegraded":"2024-09-14T04:12:02.447921+0000","last_fullsized":"2024-09-14T04:12:02.447921+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"231'12390","reported_seq":12447,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:20.754348+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:12:20.754348+0000","last_peered":"2024-09-14T04:12:20.754348+0000","last_clean":"2024-09-14T04:12:20.754348+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:12:20.754348+0000","last_undegraded":"2024-09-14T04:12:20.754348+0000","last_fullsized":"2024-09-14T04:12:20.754348+0000","mapping_epoch":184,"log_start":"199'5072","ondisk_log_start":"199'5072","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133431296,"num_objects":2036,"num_object_clones":0,"num_object_copies":6108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12390,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":435,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438449+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:12:02.438449+0000","last_peered":"2024-09-14T04:12:02.438449+0000","last_clean":"2024-09-14T04:12:02.438449+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:12:02.438449+0000","last_undegraded":"2024-09-14T04:12:02.438449+0000","last_fullsized":"2024-09-14T04:12:02.438449+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":162,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.271357+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:12:14.271357+0000","last_peered":"2024-09-14T04:12:14.271357+0000","last_clean":"2024-09-14T04:12:14.271357+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:12:14.271357+0000","last_undegraded":"2024-09-14T04:12:14.271357+0000","last_fullsized":"2024-09-14T04:12:14.271357+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":298,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296541+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:12:14.296541+0000","last_peered":"2024-09-14T04:12:14.296541+0000","last_clean":"2024-09-14T04:12:14.296541+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:12:14.296541+0000","last_undegraded":"2024-09-14T04:12:14.296541+0000","last_fullsized":"2024-09-14T04:12:14.296541+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"231'12456","reported_seq":12512,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:21.248057+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:12:21.248057+0000","last_peered":"2024-09-14T04:12:21.248057+0000","last_clean":"2024-09-14T04:12:21.248057+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:12:21.248057+0000","last_undegraded":"2024-09-14T04:12:21.248057+0000","last_fullsized":"2024-09-14T04:12:21.248057+0000","mapping_epoch":184,"log_start":"206'5900","ondisk_log_start":"206'5900","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132513792,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":600,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450612+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:12:02.450612+0000","last_peered":"2024-09-14T04:12:02.450612+0000","last_clean":"2024-09-14T04:12:02.450612+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:12:02.450612+0000","last_undegraded":"2024-09-14T04:12:02.450612+0000","last_fullsized":"2024-09-14T04:12:02.450612+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":214,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.439032+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:12:02.439032+0000","last_peered":"2024-09-14T04:12:02.439032+0000","last_clean":"2024-09-14T04:12:02.439032+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:12:02.439032+0000","last_undegraded":"2024-09-14T04:12:02.439032+0000","last_fullsized":"2024-09-14T04:12:02.439032+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":161,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438527+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:12:02.438527+0000","last_peered":"2024-09-14T04:12:02.438527+0000","last_clean":"2024-09-14T04:12:02.438527+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:12:02.438527+0000","last_undegraded":"2024-09-14T04:12:02.438527+0000","last_fullsized":"2024-09-14T04:12:02.438527+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.443424+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:12:02.443424+0000","last_peered":"2024-09-14T04:12:02.443424+0000","last_clean":"2024-09-14T04:12:02.443424+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:12:02.443424+0000","last_undegraded":"2024-09-14T04:12:02.443424+0000","last_fullsized":"2024-09-14T04:12:02.443424+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"231'12013","reported_seq":12119,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:18.702627+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:12:18.702627+0000","last_peered":"2024-09-14T04:12:18.702627+0000","last_clean":"2024-09-14T04:12:18.702627+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:12:18.702627+0000","last_undegraded":"2024-09-14T04:12:18.702627+0000","last_fullsized":"2024-09-14T04:12:18.702627+0000","mapping_epoch":184,"log_start":"199'4959","ondisk_log_start":"199'4959","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12013,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":481,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296760+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:12:14.296760+0000","last_peered":"2024-09-14T04:12:14.296760+0000","last_clean":"2024-09-14T04:12:14.296760+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:12:14.296760+0000","last_undegraded":"2024-09-14T04:12:14.296760+0000","last_fullsized":"2024-09-14T04:12:14.296760+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":244,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450462+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:12:02.450462+0000","last_peered":"2024-09-14T04:12:02.450462+0000","last_clean":"2024-09-14T04:12:02.450462+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:12:02.450462+0000","last_undegraded":"2024-09-14T04:12:02.450462+0000","last_fullsized":"2024-09-14T04:12:02.450462+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":277,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.272550+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:12:14.272550+0000","last_peered":"2024-09-14T04:12:14.272550+0000","last_clean":"2024-09-14T04:12:14.272550+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:12:14.272550+0000","last_undegraded":"2024-09-14T04:12:14.272550+0000","last_fullsized":"2024-09-14T04:12:14.272550+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269383+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:12:14.269383+0000","last_peered":"2024-09-14T04:12:14.269383+0000","last_clean":"2024-09-14T04:12:14.269383+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:12:14.269383+0000","last_undegraded":"2024-09-14T04:12:14.269383+0000","last_fullsized":"2024-09-14T04:12:14.269383+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"231'12455","reported_seq":15790,"reported_epoch":231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:22.589854+0000","last_change":"2024-09-14T04:10:35.403730+0000","last_active":"2024-09-14T04:12:22.589854+0000","last_peered":"2024-09-14T04:12:22.589854+0000","last_clean":"2024-09-14T04:09:49.197821+0000","last_became_active":"2024-09-14T04:09:51.414801+0000","last_became_peered":"2024-09-14T04:09:51.414801+0000","last_unstale":"2024-09-14T04:12:22.589854+0000","last_undegraded":"2024-09-14T04:10:35.401112+0000","last_fullsized":"2024-09-14T04:09:51.215769+0000","mapping_epoch":202,"log_start":"199'4100","ondisk_log_start":"199'4100","created":184,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8355,"log_dups_size":0,"ondisk_log_size":8355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12455,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":83},{"shards":"2,7","objects":-2313}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":463,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450176+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:12:02.450176+0000","last_peered":"2024-09-14T04:12:02.450176+0000","last_clean":"2024-09-14T04:12:02.450176+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:12:02.450176+0000","last_undegraded":"2024-09-14T04:12:02.450176+0000","last_fullsized":"2024-09-14T04:12:02.450176+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":341,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276687+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:12:14.276687+0000","last_peered":"2024-09-14T04:12:14.276687+0000","last_clean":"2024-09-14T04:12:14.276687+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:12:14.276687+0000","last_undegraded":"2024-09-14T04:12:14.276687+0000","last_fullsized":"2024-09-14T04:12:14.276687+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":284,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.295265+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:12:14.295265+0000","last_peered":"2024-09-14T04:12:14.295265+0000","last_clean":"2024-09-14T04:12:14.295265+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:12:14.295265+0000","last_undegraded":"2024-09-14T04:12:14.295265+0000","last_fullsized":"2024-09-14T04:12:14.295265+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":352,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296364+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:12:14.296364+0000","last_peered":"2024-09-14T04:12:14.296364+0000","last_clean":"2024-09-14T04:12:14.296364+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:12:14.296364+0000","last_undegraded":"2024-09-14T04:12:14.296364+0000","last_fullsized":"2024-09-14T04:12:14.296364+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"231'12185","reported_seq":13466,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:21.264227+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:12:21.264227+0000","last_peered":"2024-09-14T04:12:21.264227+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:12:21.264227+0000","last_undegraded":"2024-09-14T04:12:21.264227+0000","last_fullsized":"2024-09-14T04:12:21.264227+0000","mapping_epoch":210,"log_start":"205'5600","ondisk_log_start":"205'5600","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1999,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12185,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1999}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":469,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.439408+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:12:02.439408+0000","last_peered":"2024-09-14T04:12:02.439408+0000","last_clean":"2024-09-14T04:12:02.439408+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:12:02.439408+0000","last_undegraded":"2024-09-14T04:12:02.439408+0000","last_fullsized":"2024-09-14T04:12:02.439408+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":215,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438677+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:12:02.438677+0000","last_peered":"2024-09-14T04:12:02.438677+0000","last_clean":"2024-09-14T04:12:02.438677+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:12:02.438677+0000","last_undegraded":"2024-09-14T04:12:02.438677+0000","last_fullsized":"2024-09-14T04:12:02.438677+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":234,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446103+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:12:02.446103+0000","last_peered":"2024-09-14T04:12:02.446103+0000","last_clean":"2024-09-14T04:12:02.446103+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:12:02.446103+0000","last_undegraded":"2024-09-14T04:12:02.446103+0000","last_fullsized":"2024-09-14T04:12:02.446103+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":229,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438556+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:12:02.438556+0000","last_peered":"2024-09-14T04:12:02.438556+0000","last_clean":"2024-09-14T04:12:02.438556+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:12:02.438556+0000","last_undegraded":"2024-09-14T04:12:02.438556+0000","last_fullsized":"2024-09-14T04:12:02.438556+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"231'12551","reported_seq":12608,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:21.094014+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:12:21.094014+0000","last_peered":"2024-09-14T04:12:21.094014+0000","last_clean":"2024-09-14T04:12:21.094014+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:12:21.094014+0000","last_undegraded":"2024-09-14T04:12:21.094014+0000","last_fullsized":"2024-09-14T04:12:21.094014+0000","mapping_epoch":184,"log_start":"199'4600","ondisk_log_start":"199'4600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133758976,"num_objects":2041,"num_object_clones":0,"num_object_copies":6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12551,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":443,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.447680+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:12:02.447680+0000","last_peered":"2024-09-14T04:12:02.447680+0000","last_clean":"2024-09-14T04:12:02.447680+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:12:02.447680+0000","last_undegraded":"2024-09-14T04:12:02.447680+0000","last_fullsized":"2024-09-14T04:12:02.447680+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":239,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438818+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:12:02.438818+0000","last_peered":"2024-09-14T04:12:02.438818+0000","last_clean":"2024-09-14T04:12:02.438818+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:12:02.438818+0000","last_undegraded":"2024-09-14T04:12:02.438818+0000","last_fullsized":"2024-09-14T04:12:02.438818+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":352,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296892+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:12:14.296892+0000","last_peered":"2024-09-14T04:12:14.296892+0000","last_clean":"2024-09-14T04:12:14.296892+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:12:14.296892+0000","last_undegraded":"2024-09-14T04:12:14.296892+0000","last_fullsized":"2024-09-14T04:12:14.296892+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":301,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296488+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:12:14.296488+0000","last_peered":"2024-09-14T04:12:14.296488+0000","last_clean":"2024-09-14T04:12:14.296488+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:12:14.296488+0000","last_undegraded":"2024-09-14T04:12:14.296488+0000","last_fullsized":"2024-09-14T04:12:14.296488+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"231'12331","reported_seq":13528,"reported_epoch":231,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:21.261597+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:12:21.261597+0000","last_peered":"2024-09-14T04:12:21.261597+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:12:21.261597+0000","last_undegraded":"2024-09-14T04:12:21.261597+0000","last_fullsized":"2024-09-14T04:12:21.261597+0000","mapping_epoch":210,"log_start":"205'5800","ondisk_log_start":"205'5800","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712728,"num_objects":1965,"num_object_clones":0,"num_object_copies":5895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1965,"num_objects_unfound":0,"num_objects_dirty":1965,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12331,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":1965}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":477,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.442807+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:12:02.442807+0000","last_peered":"2024-09-14T04:12:02.442807+0000","last_clean":"2024-09-14T04:12:02.442807+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:12:02.442807+0000","last_undegraded":"2024-09-14T04:12:02.442807+0000","last_fullsized":"2024-09-14T04:12:02.442807+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":283,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296782+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:12:14.296782+0000","last_peered":"2024-09-14T04:12:14.296782+0000","last_clean":"2024-09-14T04:12:14.296782+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:12:14.296782+0000","last_undegraded":"2024-09-14T04:12:14.296782+0000","last_fullsized":"2024-09-14T04:12:14.296782+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":246,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451559+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:12:02.451559+0000","last_peered":"2024-09-14T04:12:02.451559+0000","last_clean":"2024-09-14T04:12:02.451559+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:12:02.451559+0000","last_undegraded":"2024-09-14T04:12:02.451559+0000","last_fullsized":"2024-09-14T04:12:02.451559+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269757+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:12:14.269757+0000","last_peered":"2024-09-14T04:12:14.269757+0000","last_clean":"2024-09-14T04:12:14.269757+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:12:14.269757+0000","last_undegraded":"2024-09-14T04:12:14.269757+0000","last_fullsized":"2024-09-14T04:12:14.269757+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"231'12220","reported_seq":12277,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:20.746431+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:12:20.746431+0000","last_peered":"2024-09-14T04:12:20.746431+0000","last_clean":"2024-09-14T04:12:20.746431+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:12:20.746431+0000","last_undegraded":"2024-09-14T04:12:20.746431+0000","last_fullsized":"2024-09-14T04:12:20.746431+0000","mapping_epoch":184,"log_start":"199'4812","ondisk_log_start":"199'4812","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131727360,"num_objects":2010,"num_object_clones":0,"num_object_copies":6030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12220,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":435,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445305+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:12:02.445305+0000","last_peered":"2024-09-14T04:12:02.445305+0000","last_clean":"2024-09-14T04:12:02.445305+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:12:02.445305+0000","last_undegraded":"2024-09-14T04:12:02.445305+0000","last_fullsized":"2024-09-14T04:12:02.445305+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":164,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.451312+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:12:02.451312+0000","last_peered":"2024-09-14T04:12:02.451312+0000","last_clean":"2024-09-14T04:12:02.451312+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:12:02.451312+0000","last_undegraded":"2024-09-14T04:12:02.451312+0000","last_fullsized":"2024-09-14T04:12:02.451312+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":284,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296210+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:12:14.296210+0000","last_peered":"2024-09-14T04:12:14.296210+0000","last_clean":"2024-09-14T04:12:14.296210+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:12:14.296210+0000","last_undegraded":"2024-09-14T04:12:14.296210+0000","last_fullsized":"2024-09-14T04:12:14.296210+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273869+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:12:14.273869+0000","last_peered":"2024-09-14T04:12:14.273869+0000","last_clean":"2024-09-14T04:12:14.273869+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:12:14.273869+0000","last_undegraded":"2024-09-14T04:12:14.273869+0000","last_fullsized":"2024-09-14T04:12:14.273869+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"231'12301","reported_seq":12358,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:21.252036+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:12:21.252036+0000","last_peered":"2024-09-14T04:12:21.252036+0000","last_clean":"2024-09-14T04:12:21.252036+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:12:21.252036+0000","last_undegraded":"2024-09-14T04:12:21.252036+0000","last_fullsized":"2024-09-14T04:12:21.252036+0000","mapping_epoch":184,"log_start":"205'5700","ondisk_log_start":"205'5700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12301,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":509,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275450+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:12:14.275450+0000","last_peered":"2024-09-14T04:12:14.275450+0000","last_clean":"2024-09-14T04:12:14.275450+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:12:14.275450+0000","last_undegraded":"2024-09-14T04:12:14.275450+0000","last_fullsized":"2024-09-14T04:12:14.275450+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":58,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269195+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:12:14.269195+0000","last_peered":"2024-09-14T04:12:14.269195+0000","last_clean":"2024-09-14T04:12:14.269195+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:12:14.269195+0000","last_undegraded":"2024-09-14T04:12:14.269195+0000","last_fullsized":"2024-09-14T04:12:14.269195+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":229,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450567+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:12:02.450567+0000","last_peered":"2024-09-14T04:12:02.450567+0000","last_clean":"2024-09-14T04:12:02.450567+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:12:02.450567+0000","last_undegraded":"2024-09-14T04:12:02.450567+0000","last_fullsized":"2024-09-14T04:12:02.450567+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":185,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446552+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:12:02.446552+0000","last_peered":"2024-09-14T04:12:02.446552+0000","last_clean":"2024-09-14T04:12:02.446552+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:12:02.446552+0000","last_undegraded":"2024-09-14T04:12:02.446552+0000","last_fullsized":"2024-09-14T04:12:02.446552+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":207,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445895+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:12:02.445895+0000","last_peered":"2024-09-14T04:12:02.445895+0000","last_clean":"2024-09-14T04:12:02.445895+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:12:02.445895+0000","last_undegraded":"2024-09-14T04:12:02.445895+0000","last_fullsized":"2024-09-14T04:12:02.445895+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":207,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.446440+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:12:02.446440+0000","last_peered":"2024-09-14T04:12:02.446440+0000","last_clean":"2024-09-14T04:12:02.446440+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:12:02.446440+0000","last_undegraded":"2024-09-14T04:12:02.446440+0000","last_fullsized":"2024-09-14T04:12:02.446440+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":284,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.296361+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:12:14.296361+0000","last_peered":"2024-09-14T04:12:14.296361+0000","last_clean":"2024-09-14T04:12:14.296361+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:12:14.296361+0000","last_undegraded":"2024-09-14T04:12:14.296361+0000","last_fullsized":"2024-09-14T04:12:14.296361+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":238,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450668+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:12:02.450668+0000","last_peered":"2024-09-14T04:12:02.450668+0000","last_clean":"2024-09-14T04:12:02.450668+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:12:02.450668+0000","last_undegraded":"2024-09-14T04:12:02.450668+0000","last_fullsized":"2024-09-14T04:12:02.450668+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":199,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.439131+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:12:02.439131+0000","last_peered":"2024-09-14T04:12:02.439131+0000","last_clean":"2024-09-14T04:12:02.439131+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:12:02.439131+0000","last_undegraded":"2024-09-14T04:12:02.439131+0000","last_fullsized":"2024-09-14T04:12:02.439131+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":161,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.454463+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:12:02.454463+0000","last_peered":"2024-09-14T04:12:02.454463+0000","last_clean":"2024-09-14T04:12:02.454463+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:12:02.454463+0000","last_undegraded":"2024-09-14T04:12:02.454463+0000","last_fullsized":"2024-09-14T04:12:02.454463+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269105+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:12:14.269105+0000","last_peered":"2024-09-14T04:12:14.269105+0000","last_clean":"2024-09-14T04:12:14.269105+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:12:14.269105+0000","last_undegraded":"2024-09-14T04:12:14.269105+0000","last_fullsized":"2024-09-14T04:12:14.269105+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":162,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.271454+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:12:14.271454+0000","last_peered":"2024-09-14T04:12:14.271454+0000","last_clean":"2024-09-14T04:12:14.271454+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:12:14.271454+0000","last_undegraded":"2024-09-14T04:12:14.271454+0000","last_fullsized":"2024-09-14T04:12:14.271454+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.450418+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:12:02.450418+0000","last_peered":"2024-09-14T04:12:02.450418+0000","last_clean":"2024-09-14T04:12:02.450418+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:12:02.450418+0000","last_undegraded":"2024-09-14T04:12:02.450418+0000","last_fullsized":"2024-09-14T04:12:02.450418+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.271524+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:12:14.271524+0000","last_peered":"2024-09-14T04:12:14.271524+0000","last_clean":"2024-09-14T04:12:14.271524+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:12:14.271524+0000","last_undegraded":"2024-09-14T04:12:14.271524+0000","last_fullsized":"2024-09-14T04:12:14.271524+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269101+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:12:14.269101+0000","last_peered":"2024-09-14T04:12:14.269101+0000","last_clean":"2024-09-14T04:12:14.269101+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:12:14.269101+0000","last_undegraded":"2024-09-14T04:12:14.269101+0000","last_fullsized":"2024-09-14T04:12:14.269101+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":310,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.295414+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:12:14.295414+0000","last_peered":"2024-09-14T04:12:14.295414+0000","last_clean":"2024-09-14T04:12:14.295414+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:12:14.295414+0000","last_undegraded":"2024-09-14T04:12:14.295414+0000","last_fullsized":"2024-09-14T04:12:14.295414+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438572+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:12:02.438572+0000","last_peered":"2024-09-14T04:12:02.438572+0000","last_clean":"2024-09-14T04:12:02.438572+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:12:02.438572+0000","last_undegraded":"2024-09-14T04:12:02.438572+0000","last_fullsized":"2024-09-14T04:12:02.438572+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":208,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269164+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:12:14.269164+0000","last_peered":"2024-09-14T04:12:14.269164+0000","last_clean":"2024-09-14T04:12:14.269164+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:12:14.269164+0000","last_undegraded":"2024-09-14T04:12:14.269164+0000","last_fullsized":"2024-09-14T04:12:14.269164+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":327,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.270468+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:12:14.270468+0000","last_peered":"2024-09-14T04:12:14.270468+0000","last_clean":"2024-09-14T04:12:14.270468+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:12:14.270468+0000","last_undegraded":"2024-09-14T04:12:14.270468+0000","last_fullsized":"2024-09-14T04:12:14.270468+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":54,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269656+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:12:14.269656+0000","last_peered":"2024-09-14T04:12:14.269656+0000","last_clean":"2024-09-14T04:12:14.269656+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:12:14.269656+0000","last_undegraded":"2024-09-14T04:12:14.269656+0000","last_fullsized":"2024-09-14T04:12:14.269656+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445028+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:12:02.445028+0000","last_peered":"2024-09-14T04:12:02.445028+0000","last_clean":"2024-09-14T04:12:02.445028+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:12:02.445028+0000","last_undegraded":"2024-09-14T04:12:02.445028+0000","last_fullsized":"2024-09-14T04:12:02.445028+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":237,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438771+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:12:02.438771+0000","last_peered":"2024-09-14T04:12:02.438771+0000","last_clean":"2024-09-14T04:12:02.438771+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:12:02.438771+0000","last_undegraded":"2024-09-14T04:12:02.438771+0000","last_fullsized":"2024-09-14T04:12:02.438771+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445148+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:12:02.445148+0000","last_peered":"2024-09-14T04:12:02.445148+0000","last_clean":"2024-09-14T04:12:02.445148+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:12:02.445148+0000","last_undegraded":"2024-09-14T04:12:02.445148+0000","last_fullsized":"2024-09-14T04:12:02.445148+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":224,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.438740+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:12:02.438740+0000","last_peered":"2024-09-14T04:12:02.438740+0000","last_clean":"2024-09-14T04:12:02.438740+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:12:02.438740+0000","last_undegraded":"2024-09-14T04:12:02.438740+0000","last_fullsized":"2024-09-14T04:12:02.438740+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276535+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:12:14.276535+0000","last_peered":"2024-09-14T04:12:14.276535+0000","last_clean":"2024-09-14T04:12:14.276535+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:12:14.276535+0000","last_undegraded":"2024-09-14T04:12:14.276535+0000","last_fullsized":"2024-09-14T04:12:14.276535+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":107,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.266906+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:12:14.266906+0000","last_peered":"2024-09-14T04:12:14.266906+0000","last_clean":"2024-09-14T04:12:14.266906+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:12:14.266906+0000","last_undegraded":"2024-09-14T04:12:14.266906+0000","last_fullsized":"2024-09-14T04:12:14.266906+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":230,"state":"active+clean","last_fresh":"2024-09-14T04:12:02.445636+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:12:02.445636+0000","last_peered":"2024-09-14T04:12:02.445636+0000","last_clean":"2024-09-14T04:12:02.445636+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:12:02.445636+0000","last_undegraded":"2024-09-14T04:12:02.445636+0000","last_fullsized":"2024-09-14T04:12:02.445636+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":60,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.269205+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:12:14.269205+0000","last_peered":"2024-09-14T04:12:14.269205+0000","last_clean":"2024-09-14T04:12:14.269205+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:12:14.269205+0000","last_undegraded":"2024-09-14T04:12:14.269205+0000","last_fullsized":"2024-09-14T04:12:14.269205+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2112094232,"num_objects":32229,"num_object_clones":0,"num_object_copies":96687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9996,"num_objects_misplaced":11347,"num_objects_unfound":0,"num_objects_dirty":32229,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198047,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11894,"num_bytes_recovered":753139712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7212314624,"data_stored":7212302408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119904,"ondisk_log_size":119904,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201661,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1431904,"kb_used_data":652632,"kb_used_omap":1,"kb_used_meta":779262,"kb_avail":92313248,"statfs":{"total":95995035648,"available":94528765952,"internally_reserved":0,"allocated":668295168,"data_stored":666829619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":797964747},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463299,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1554656,"kb_used_data":914008,"kb_used_omap":1,"kb_used_meta":640638,"kb_avail":92190496,"statfs":{"total":95995035648,"available":94403067904,"internally_reserved":0,"allocated":935944192,"data_stored":934479146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":656013767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724935,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1651348,"kb_used_data":633356,"kb_used_omap":1,"kb_used_meta":1017982,"kb_avail":92093804,"statfs":{"total":95995035648,"available":94304055296,"internally_reserved":0,"allocated":648556544,"data_stored":647099279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1042414022},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986571,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2200224,"kb_used_data":1397144,"kb_used_omap":1,"kb_used_meta":803070,"kb_avail":91544928,"statfs":{"total":95995035648,"available":93742006272,"internally_reserved":0,"allocated":1430675456,"data_stored":1429209622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":822344134},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748600,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1220820,"kb_used_data":808396,"kb_used_omap":4,"kb_used_meta":412411,"kb_avail":92524332,"statfs":{"total":95995035648,"available":94744915968,"internally_reserved":0,"allocated":827797504,"data_stored":826339975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":422309420},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912127,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493972,"kb_used_data":1230732,"kb_used_omap":6,"kb_used_meta":263225,"kb_avail":92251180,"statfs":{"total":95995035648,"available":94465208320,"internally_reserved":0,"allocated":1260269568,"data_stored":1258812541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":269542969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468143,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1031432,"kb_used_data":535808,"kb_used_omap":5,"kb_used_meta":495610,"kb_avail":92713720,"statfs":{"total":95995035648,"available":94938849280,"internally_reserved":0,"allocated":548667392,"data_stored":547220092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":507504991},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342258,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152340,"kb_used_data":964172,"kb_used_omap":10,"kb_used_meta":188149,"kb_avail":92592812,"statfs":{"total":95995035648,"available":94815039488,"internally_reserved":0,"allocated":987312128,"data_stored":985854579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":192664786},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":981991424,"data_stored":981991424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":542638080,"data_stored":542638080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1248329728,"data_stored":1248329728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":804327424,"data_stored":804323352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1421479936,"data_stored":1421475864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":641925120,"data_stored":641925120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":912396288,"data_stored":912392216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":659226624,"data_stored":659226624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:12:25.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:12:25.092 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:25.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:25.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:25.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:25.337 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:25 smithi001 ceph-mon[107399]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:25.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:25 smithi012 ceph-mon[101180]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:25.635 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:25 smithi001 ceph-mon[112165]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:25.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:25.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:25.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:25.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:25.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:25.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:25.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:26.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:26.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:26.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:26.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:26.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:12:26.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:26.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:26.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:26.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.587 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:26.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:26 smithi012 ceph-mon[101180]: pgmap v798: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/94794 objects degraded (10.545%); 11347/94794 objects misplaced (11.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:26.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:26 smithi001 ceph-mon[112165]: pgmap v798: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/94794 objects degraded (10.545%); 11347/94794 objects misplaced (11.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:26.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:26 smithi001 ceph-mon[107399]: pgmap v798: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9996/94794 objects degraded (10.545%); 11347/94794 objects misplaced (11.970%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:12:26.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:26.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:26.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:26.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:26.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:26] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-09-14T04:12:26.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:27.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:27.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:27.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:27.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:27.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:27.336 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:27.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:27.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:27.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:27.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.629 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:27.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:27.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:27.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:27.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:27.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:27.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:27.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:28.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:28.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:28.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:12:28.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:28.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:28.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:28.244 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:12:28.244 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:12:28.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:28.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:28.399 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:28.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:28.464 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:28.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:28.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:28.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:28.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:28 smithi012 ceph-mon[101180]: pgmap v799: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 9844/87432 objects degraded (11.259%); 10759/87432 objects misplaced (12.306%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:28.597 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:28 smithi001 ceph-mon[112165]: pgmap v799: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 9844/87432 objects degraded (11.259%); 10759/87432 objects misplaced (12.306%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:28.597 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:28 smithi001 ceph-mon[107399]: pgmap v799: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 9844/87432 objects degraded (11.259%); 10759/87432 objects misplaced (12.306%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:28.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:28.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:28.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:28.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:28.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:28.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:28.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:29.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:29.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:29.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:29.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:29.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:29.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:29.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:29.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:12:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.452 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:29.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:29.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:29.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:29.687 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:29.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:29.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9844/87432 objects degraded (11.259%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:29.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9844/87432 objects degraded (11.259%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:29.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9844/87432 objects degraded (11.259%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:29.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:29.901 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:29.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:29.985 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:29.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:30.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:30.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:30.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:30.266 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:30.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:30.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:30.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:30.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:30.543 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[107399]: pgmap v800: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9844/84489 objects degraded (11.651%); 10759/84489 objects misplaced (12.734%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:30.543 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.544 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.544 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[112165]: pgmap v800: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9844/84489 objects degraded (11.651%); 10759/84489 objects misplaced (12.734%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:30.544 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.544 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:30.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:30.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.831 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:30 smithi012 ceph-mon[101180]: pgmap v800: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9844/84489 objects degraded (11.651%); 10759/84489 objects misplaced (12.734%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:30 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:30 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:30.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:30.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:30.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:31.068 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:12:31.069 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:12:31.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:31.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:31.112 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:12:31.113 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:12:31.144 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:12:31.145 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:12:31.196 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:12:31.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:12:31.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:31.236 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:12:31.237 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:12:31.245 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:31.277 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:12:31.277 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:12:31.311 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:12:31.311 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:12:31.342 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:12:31.343 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:12:31.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:31.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:31.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.521 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:31.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:31.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:31.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:31.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/907210336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:31.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/907210336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:31.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:31.798 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/907210336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:31.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:31.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:31.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:32.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:32.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:32.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:32.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:32.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:32.368 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:32.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:32.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:32.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.651 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:32.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:32 smithi001 ceph-mon[107399]: pgmap v801: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 9844/81009 objects degraded (12.152%); 10759/81009 objects misplaced (13.281%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:32 smithi001 ceph-mon[112165]: pgmap v801: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 9844/81009 objects degraded (12.152%); 10759/81009 objects misplaced (13.281%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:32.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:32.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:32 smithi012 ceph-mon[101180]: pgmap v801: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 9844/81009 objects degraded (12.152%); 10759/81009 objects misplaced (13.281%); 2.3 MiB/s, 51 objects/s recovering 2024-09-14T04:12:32.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:32.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:32.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:33.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:33.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:33.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:33.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:33.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:33.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:33.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:33.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:33.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:33.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:33.642 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:33.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:33.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:33.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:33.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:33.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:34.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:34.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:34.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:34.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:34.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:34.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.487 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:34.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:34.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:34.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:34 smithi001 ceph-mon[112165]: pgmap v802: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9487/73914 objects degraded (12.835%); 10281/73914 objects misplaced (13.909%); 723 KiB/s, 62 objects/s recovering 2024-09-14T04:12:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9487/73914 objects degraded (12.835%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:34.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:34 smithi001 ceph-mon[107399]: pgmap v802: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9487/73914 objects degraded (12.835%); 10281/73914 objects misplaced (13.909%); 723 KiB/s, 62 objects/s recovering 2024-09-14T04:12:34.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9487/73914 objects degraded (12.835%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:34.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:34 smithi012 ceph-mon[101180]: pgmap v802: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 9487/73914 objects degraded (12.835%); 10281/73914 objects misplaced (13.909%); 723 KiB/s, 62 objects/s recovering 2024-09-14T04:12:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9487/73914 objects degraded (12.835%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:34.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:34.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:34.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:34.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:35.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:35.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:35.130 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:35.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:35.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:35.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:35.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:35.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:35.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:35.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:12:35.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:35.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:35.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:35.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:35.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:35.882 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:35.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:35.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:36.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:36.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.155 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:36.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[107399]: pgmap v803: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9487/71667 objects degraded (13.238%); 10281/71667 objects misplaced (14.346%); 609 KiB/s, 52 objects/s recovering 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1998952503' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.163 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:36.164 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:36.164 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[112165]: pgmap v803: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9487/71667 objects degraded (13.238%); 10281/71667 objects misplaced (14.346%); 609 KiB/s, 52 objects/s recovering 2024-09-14T04:12:36.164 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1998952503' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.164 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:35 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.256 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:36.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:36.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:35 smithi012 ceph-mon[101180]: pgmap v803: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9487/71667 objects degraded (13.238%); 10281/71667 objects misplaced (14.346%); 609 KiB/s, 52 objects/s recovering 2024-09-14T04:12:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1998952503' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:35 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:12:36.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:36.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:36.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:36.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:36.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:36.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:36.800 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:36.804 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:12:36.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:36.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:36.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:36.932 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:12:36.935 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:36] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-09-14T04:12:36.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:37.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:36 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:36 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:12:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:36 smithi012 ceph-mon[101180]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T04:12:37.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:37.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:37.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:37.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:12:37.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[107399]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T04:12:37.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:37.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:12:37.223 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:36 smithi001 ceph-mon[112165]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T04:12:37.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:37.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:37.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:37.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:37.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:37.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:37.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:37.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:37.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:37.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:37.865 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:37.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:37.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:38.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:38.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:38.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:38.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.139 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:38.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:37 smithi001 ceph-mon[112165]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:37 smithi001 ceph-mon[112165]: pgmap v805: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9076/64086 objects degraded (14.162%); 9766/64086 objects misplaced (15.239%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:37 smithi001 ceph-mon[107399]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:38.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:37 smithi001 ceph-mon[107399]: pgmap v805: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9076/64086 objects degraded (14.162%); 9766/64086 objects misplaced (15.239%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:38.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:38.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:37 smithi012 ceph-mon[101180]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:12:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:37 smithi012 ceph-mon[101180]: pgmap v805: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9076/64086 objects degraded (14.162%); 9766/64086 objects misplaced (15.239%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:38.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:38.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:38.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:38.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:38.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:38.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:38.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:38.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:38.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:38.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:38.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:39.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:39.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:39.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:39.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:39.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9076/64086 objects degraded (14.162%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:39.366 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:39.468 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:39.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9076/64086 objects degraded (14.162%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:39.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9076/64086 objects degraded (14.162%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:39.512 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:39.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:39.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:39.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:39.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:39.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:39.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:39.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:39.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:40.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:40.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:40.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:40.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:40.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:40.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:40.342 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:40 smithi012 ceph-mon[101180]: pgmap v806: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9076/61029 objects degraded (14.872%); 9766/61029 objects misplaced (16.002%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:40.372 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:40 smithi001 ceph-mon[112165]: pgmap v806: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9076/61029 objects degraded (14.872%); 9766/61029 objects misplaced (16.002%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:40.372 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:40 smithi001 ceph-mon[107399]: pgmap v806: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9076/61029 objects degraded (14.872%); 9766/61029 objects misplaced (16.002%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:40.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:40.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:40.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:40.624 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:40.726 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:40.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:40.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:40.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:40.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:40.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:41.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:41.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:41.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:41.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:41.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:41.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:41.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.475 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:41.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:41.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:41.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:41.718 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:41.760 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:41.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:41.900 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:41.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:42.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:42.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:42.059 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":804,"stamp":"2024-09-14T04:12:35.935279+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1533614077,"num_objects":23419,"num_object_clones":0,"num_object_copies":70257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9487,"num_objects_misplaced":10281,"num_objects_unfound":0,"num_objects_dirty":23419,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":206947,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12554,"num_bytes_recovered":761464274,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121638,"ondisk_log_size":121638,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10020024,"kb_used_data":5379192,"kb_used_omap":33,"kb_used_meta":4640478,"kb_avail":739941192,"statfs":{"total":767960285184,"available":757699780608,"internally_reserved":0,"allocated":5508292608,"data_stored":5496557397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4751849908},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,4,1,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536018944,"num_objects":-8179,"num_object_clones":0,"num_object_copies":-24537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-509,"num_objects_misplaced":-1066,"num_objects_unfound":0,"num_objects_dirty":-8179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8179,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":622,"num_bytes_recovered":7405568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.750386"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.329877+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:12:23.329877+0000","last_peered":"2024-09-14T04:12:23.329877+0000","last_clean":"2024-09-14T04:12:23.329877+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:12:23.329877+0000","last_undegraded":"2024-09-14T04:12:23.329877+0000","last_fullsized":"2024-09-14T04:12:23.329877+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":206,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.328102+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:12:23.328102+0000","last_peered":"2024-09-14T04:12:23.328102+0000","last_clean":"2024-09-14T04:12:23.328102+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:12:23.328102+0000","last_undegraded":"2024-09-14T04:12:23.328102+0000","last_fullsized":"2024-09-14T04:12:23.328102+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":42,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:16.789771+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:12:16.789771+0000","last_peered":"2024-09-14T04:12:16.789771+0000","last_clean":"2024-09-14T04:12:16.789771+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:12:16.789771+0000","last_undegraded":"2024-09-14T04:12:16.789771+0000","last_fullsized":"2024-09-14T04:12:16.789771+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.326104+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:12:23.326104+0000","last_peered":"2024-09-14T04:12:23.326104+0000","last_clean":"2024-09-14T04:12:23.326104+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:12:23.326104+0000","last_undegraded":"2024-09-14T04:12:23.326104+0000","last_fullsized":"2024-09-14T04:12:23.326104+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325897+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:12:23.325897+0000","last_peered":"2024-09-14T04:12:23.325897+0000","last_clean":"2024-09-14T04:12:23.325897+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:12:23.325897+0000","last_undegraded":"2024-09-14T04:12:23.325897+0000","last_fullsized":"2024-09-14T04:12:23.325897+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":185,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275420+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:12:14.275420+0000","last_peered":"2024-09-14T04:12:14.275420+0000","last_clean":"2024-09-14T04:12:14.275420+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:12:14.275420+0000","last_undegraded":"2024-09-14T04:12:14.275420+0000","last_fullsized":"2024-09-14T04:12:14.275420+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.345306+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:12:23.345306+0000","last_peered":"2024-09-14T04:12:23.345306+0000","last_clean":"2024-09-14T04:12:23.345306+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:12:23.345306+0000","last_undegraded":"2024-09-14T04:12:23.345306+0000","last_fullsized":"2024-09-14T04:12:23.345306+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":282,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.266303+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:12:14.266303+0000","last_peered":"2024-09-14T04:12:14.266303+0000","last_clean":"2024-09-14T04:12:14.266303+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:12:14.266303+0000","last_undegraded":"2024-09-14T04:12:14.266303+0000","last_fullsized":"2024-09-14T04:12:14.266303+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"232'13247","reported_seq":13305,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:31.809590+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:12:31.809590+0000","last_peered":"2024-09-14T04:12:31.809590+0000","last_clean":"2024-09-14T04:12:31.809590+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:12:31.809590+0000","last_undegraded":"2024-09-14T04:12:31.809590+0000","last_fullsized":"2024-09-14T04:12:31.809590+0000","mapping_epoch":184,"log_start":"198'3500","ondisk_log_start":"198'3500","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9747,"log_dups_size":0,"ondisk_log_size":9747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102694912,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13247,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":243,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.274862+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:12:14.274862+0000","last_peered":"2024-09-14T04:12:14.274862+0000","last_clean":"2024-09-14T04:12:14.274862+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:12:14.274862+0000","last_undegraded":"2024-09-14T04:12:14.274862+0000","last_fullsized":"2024-09-14T04:12:14.274862+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273086+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:12:14.273086+0000","last_peered":"2024-09-14T04:12:14.273086+0000","last_clean":"2024-09-14T04:12:14.273086+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:12:14.273086+0000","last_undegraded":"2024-09-14T04:12:14.273086+0000","last_fullsized":"2024-09-14T04:12:14.273086+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"232'13044","reported_seq":22457,"reported_epoch":232,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:31.329846+0000","last_change":"2024-09-14T04:10:35.400317+0000","last_active":"2024-09-14T04:12:31.329846+0000","last_peered":"2024-09-14T04:12:31.329846+0000","last_clean":"2024-09-14T04:09:49.203876+0000","last_became_active":"2024-09-14T04:09:51.437466+0000","last_became_peered":"2024-09-14T04:09:51.437466+0000","last_unstale":"2024-09-14T04:12:31.329846+0000","last_undegraded":"2024-09-14T04:09:51.253655+0000","last_fullsized":"2024-09-14T04:09:51.216149+0000","mapping_epoch":202,"log_start":"207'6500","ondisk_log_start":"207'6500","created":184,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:19:14.518927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13044,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4664,"num_bytes_recovered":272302080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"3,6","objects":170}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":242,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.274520+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:12:14.274520+0000","last_peered":"2024-09-14T04:12:14.274520+0000","last_clean":"2024-09-14T04:12:14.274520+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:12:14.274520+0000","last_undegraded":"2024-09-14T04:12:14.274520+0000","last_fullsized":"2024-09-14T04:12:14.274520+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":233,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.278072+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:12:14.278072+0000","last_peered":"2024-09-14T04:12:14.278072+0000","last_clean":"2024-09-14T04:12:14.278072+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:12:14.278072+0000","last_undegraded":"2024-09-14T04:12:14.278072+0000","last_fullsized":"2024-09-14T04:12:14.278072+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"232'13181","reported_seq":14477,"reported_epoch":232,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:12:34.893352+0000","last_change":"2024-09-14T04:10:36.624348+0000","last_active":"2024-09-14T04:12:34.893352+0000","last_peered":"2024-09-14T04:12:34.893352+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:12:34.893352+0000","last_undegraded":"2024-09-14T04:12:34.893352+0000","last_fullsized":"2024-09-14T04:12:34.893352+0000","mapping_epoch":210,"log_start":"199'5200","ondisk_log_start":"199'5200","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7981,"log_dups_size":0,"ondisk_log_size":7981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194496,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7270,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13181,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":594,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-5502}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":213,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275141+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:12:14.275141+0000","last_peered":"2024-09-14T04:12:14.275141+0000","last_clean":"2024-09-14T04:12:14.275141+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:12:14.275141+0000","last_undegraded":"2024-09-14T04:12:14.275141+0000","last_fullsized":"2024-09-14T04:12:14.275141+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":198,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.265482+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:12:14.265482+0000","last_peered":"2024-09-14T04:12:14.265482+0000","last_clean":"2024-09-14T04:12:14.265482+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:12:14.265482+0000","last_undegraded":"2024-09-14T04:12:14.265482+0000","last_fullsized":"2024-09-14T04:12:14.265482+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"232'13017","reported_seq":13125,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:32.621909+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:12:32.621909+0000","last_peered":"2024-09-14T04:12:32.621909+0000","last_clean":"2024-09-14T04:12:32.621909+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:12:32.621909+0000","last_undegraded":"2024-09-14T04:12:32.621909+0000","last_fullsized":"2024-09-14T04:12:32.621909+0000","mapping_epoch":184,"log_start":"199'4600","ondisk_log_start":"199'4600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13017,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":219,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273605+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:12:14.273605+0000","last_peered":"2024-09-14T04:12:14.273605+0000","last_clean":"2024-09-14T04:12:14.273605+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:12:14.273605+0000","last_undegraded":"2024-09-14T04:12:14.273605+0000","last_fullsized":"2024-09-14T04:12:14.273605+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.328215+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:12:23.328215+0000","last_peered":"2024-09-14T04:12:23.328215+0000","last_clean":"2024-09-14T04:12:23.328215+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:12:23.328215+0000","last_undegraded":"2024-09-14T04:12:23.328215+0000","last_fullsized":"2024-09-14T04:12:23.328215+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"232'12817","reported_seq":19694,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:34.911843+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:12:34.911843+0000","last_peered":"2024-09-14T04:12:34.911843+0000","last_clean":"2024-09-14T04:12:34.911843+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:12:34.911843+0000","last_undegraded":"2024-09-14T04:12:34.911843+0000","last_fullsized":"2024-09-14T04:12:34.911843+0000","mapping_epoch":217,"log_start":"199'4900","ondisk_log_start":"199'4900","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7917,"log_dups_size":0,"ondisk_log_size":7917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89456640,"num_objects":1365,"num_object_clones":0,"num_object_copies":4095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12817,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":584,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275228+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:12:14.275228+0000","last_peered":"2024-09-14T04:12:14.275228+0000","last_clean":"2024-09-14T04:12:14.275228+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:12:14.275228+0000","last_undegraded":"2024-09-14T04:12:14.275228+0000","last_fullsized":"2024-09-14T04:12:14.275228+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":60,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325959+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:12:23.325959+0000","last_peered":"2024-09-14T04:12:23.325959+0000","last_clean":"2024-09-14T04:12:23.325959+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:12:23.325959+0000","last_undegraded":"2024-09-14T04:12:23.325959+0000","last_fullsized":"2024-09-14T04:12:23.325959+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":218,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.278573+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:12:14.278573+0000","last_peered":"2024-09-14T04:12:14.278573+0000","last_clean":"2024-09-14T04:12:14.278573+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:12:14.278573+0000","last_undegraded":"2024-09-14T04:12:14.278573+0000","last_fullsized":"2024-09-14T04:12:14.278573+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"232'12877","reported_seq":12985,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:32.605784+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:12:32.605784+0000","last_peered":"2024-09-14T04:12:32.605784+0000","last_clean":"2024-09-14T04:12:32.605784+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:12:32.605784+0000","last_undegraded":"2024-09-14T04:12:32.605784+0000","last_fullsized":"2024-09-14T04:12:32.605784+0000","mapping_epoch":184,"log_start":"199'4500","ondisk_log_start":"199'4500","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8377,"log_dups_size":0,"ondisk_log_size":8377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12877,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":453,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.272934+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:12:14.272934+0000","last_peered":"2024-09-14T04:12:14.272934+0000","last_clean":"2024-09-14T04:12:14.272934+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:12:14.272934+0000","last_undegraded":"2024-09-14T04:12:14.272934+0000","last_fullsized":"2024-09-14T04:12:14.272934+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":234,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.265824+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:12:14.265824+0000","last_peered":"2024-09-14T04:12:14.265824+0000","last_clean":"2024-09-14T04:12:14.265824+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:12:14.265824+0000","last_undegraded":"2024-09-14T04:12:14.265824+0000","last_fullsized":"2024-09-14T04:12:14.265824+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277880+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:12:14.277880+0000","last_peered":"2024-09-14T04:12:14.277880+0000","last_clean":"2024-09-14T04:12:14.277880+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:12:14.277880+0000","last_undegraded":"2024-09-14T04:12:14.277880+0000","last_fullsized":"2024-09-14T04:12:14.277880+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"232'12745","reported_seq":15055,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:31.295264+0000","last_change":"2024-09-14T04:11:13.382242+0000","last_active":"2024-09-14T04:12:31.295264+0000","last_peered":"2024-09-14T04:12:31.295264+0000","last_clean":"2024-09-14T04:09:49.209154+0000","last_became_active":"2024-09-14T04:11:13.358760+0000","last_became_peered":"2024-09-14T04:11:13.358760+0000","last_unstale":"2024-09-14T04:12:31.295264+0000","last_undegraded":"2024-09-14T04:11:13.259138+0000","last_fullsized":"2024-09-14T04:11:13.243084+0000","mapping_epoch":220,"log_start":"207'6200","ondisk_log_start":"207'6200","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:46.130184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100073472,"num_objects":1527,"num_object_clones":0,"num_object_copies":4581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3837,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12745,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":73203712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":-331},{"shards":"1,6,7","objects":1527}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":22,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277572+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:12:14.277572+0000","last_peered":"2024-09-14T04:12:14.277572+0000","last_clean":"2024-09-14T04:12:14.277572+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:12:14.277572+0000","last_undegraded":"2024-09-14T04:12:14.277572+0000","last_fullsized":"2024-09-14T04:12:14.277572+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":200,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.278554+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:12:14.278554+0000","last_peered":"2024-09-14T04:12:14.278554+0000","last_clean":"2024-09-14T04:12:14.278554+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:12:14.278554+0000","last_undegraded":"2024-09-14T04:12:14.278554+0000","last_fullsized":"2024-09-14T04:12:14.278554+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"232'13115","reported_seq":13173,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:35.792017+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:12:35.792017+0000","last_peered":"2024-09-14T04:12:35.792017+0000","last_clean":"2024-09-14T04:12:35.792017+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:12:35.792017+0000","last_undegraded":"2024-09-14T04:12:35.792017+0000","last_fullsized":"2024-09-14T04:12:35.792017+0000","mapping_epoch":184,"log_start":"205'5772","ondisk_log_start":"205'5772","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13115,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":436,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273504+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:12:14.273504+0000","last_peered":"2024-09-14T04:12:14.273504+0000","last_clean":"2024-09-14T04:12:14.273504+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:12:14.273504+0000","last_undegraded":"2024-09-14T04:12:14.273504+0000","last_fullsized":"2024-09-14T04:12:14.273504+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":164,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.337633+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:12:23.337633+0000","last_peered":"2024-09-14T04:12:23.337633+0000","last_clean":"2024-09-14T04:12:23.337633+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:12:23.337633+0000","last_undegraded":"2024-09-14T04:12:23.337633+0000","last_fullsized":"2024-09-14T04:12:23.337633+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":300,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.332933+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:12:23.332933+0000","last_peered":"2024-09-14T04:12:23.332933+0000","last_clean":"2024-09-14T04:12:23.332933+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:12:23.332933+0000","last_undegraded":"2024-09-14T04:12:23.332933+0000","last_fullsized":"2024-09-14T04:12:23.332933+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"232'12949","reported_seq":13007,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:31.323725+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:12:31.323725+0000","last_peered":"2024-09-14T04:12:31.323725+0000","last_clean":"2024-09-14T04:12:31.323725+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:12:31.323725+0000","last_undegraded":"2024-09-14T04:12:31.323725+0000","last_fullsized":"2024-09-14T04:12:31.323725+0000","mapping_epoch":184,"log_start":"207'6400","ondisk_log_start":"207'6400","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12949,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":601,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.274148+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:12:14.274148+0000","last_peered":"2024-09-14T04:12:14.274148+0000","last_clean":"2024-09-14T04:12:14.274148+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:12:14.274148+0000","last_undegraded":"2024-09-14T04:12:14.274148+0000","last_fullsized":"2024-09-14T04:12:14.274148+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":215,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.280301+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:12:14.280301+0000","last_peered":"2024-09-14T04:12:14.280301+0000","last_clean":"2024-09-14T04:12:14.280301+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:12:14.280301+0000","last_undegraded":"2024-09-14T04:12:14.280301+0000","last_fullsized":"2024-09-14T04:12:14.280301+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.331593+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:12:23.331593+0000","last_peered":"2024-09-14T04:12:23.331593+0000","last_clean":"2024-09-14T04:12:23.331593+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:12:23.331593+0000","last_undegraded":"2024-09-14T04:12:23.331593+0000","last_fullsized":"2024-09-14T04:12:23.331593+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":184,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.266449+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:12:14.266449+0000","last_peered":"2024-09-14T04:12:14.266449+0000","last_clean":"2024-09-14T04:12:14.266449+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:12:14.266449+0000","last_undegraded":"2024-09-14T04:12:14.266449+0000","last_fullsized":"2024-09-14T04:12:14.266449+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"232'12721","reported_seq":12829,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:33.788299+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:12:33.788299+0000","last_peered":"2024-09-14T04:12:33.788299+0000","last_clean":"2024-09-14T04:12:33.788299+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:12:33.788299+0000","last_undegraded":"2024-09-14T04:12:33.788299+0000","last_fullsized":"2024-09-14T04:12:33.788299+0000","mapping_epoch":184,"log_start":"205'5659","ondisk_log_start":"205'5659","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92602368,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12721,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":483,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.332948+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:12:23.332948+0000","last_peered":"2024-09-14T04:12:23.332948+0000","last_clean":"2024-09-14T04:12:23.332948+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:12:23.332948+0000","last_undegraded":"2024-09-14T04:12:23.332948+0000","last_fullsized":"2024-09-14T04:12:23.332948+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":245,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.274077+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:12:14.274077+0000","last_peered":"2024-09-14T04:12:14.274077+0000","last_clean":"2024-09-14T04:12:14.274077+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:12:14.274077+0000","last_undegraded":"2024-09-14T04:12:14.274077+0000","last_fullsized":"2024-09-14T04:12:14.274077+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":280,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325628+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:12:23.325628+0000","last_peered":"2024-09-14T04:12:23.325628+0000","last_clean":"2024-09-14T04:12:23.325628+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:12:23.325628+0000","last_undegraded":"2024-09-14T04:12:23.325628+0000","last_fullsized":"2024-09-14T04:12:23.325628+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.326109+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:12:23.326109+0000","last_peered":"2024-09-14T04:12:23.326109+0000","last_clean":"2024-09-14T04:12:23.326109+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:12:23.326109+0000","last_undegraded":"2024-09-14T04:12:23.326109+0000","last_fullsized":"2024-09-14T04:12:23.326109+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"232'12979","reported_seq":16314,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:32.660285+0000","last_change":"2024-09-14T04:10:35.403730+0000","last_active":"2024-09-14T04:12:32.660285+0000","last_peered":"2024-09-14T04:12:32.660285+0000","last_clean":"2024-09-14T04:09:49.197821+0000","last_became_active":"2024-09-14T04:09:51.414801+0000","last_became_peered":"2024-09-14T04:09:51.414801+0000","last_unstale":"2024-09-14T04:12:32.660285+0000","last_undegraded":"2024-09-14T04:10:35.401112+0000","last_fullsized":"2024-09-14T04:09:51.215769+0000","mapping_epoch":202,"log_start":"199'4600","ondisk_log_start":"199'4600","created":184,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8379,"log_dups_size":0,"ondisk_log_size":8379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12979,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,7","objects":-2837}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":464,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273451+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:12:14.273451+0000","last_peered":"2024-09-14T04:12:14.273451+0000","last_clean":"2024-09-14T04:12:14.273451+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:12:14.273451+0000","last_undegraded":"2024-09-14T04:12:14.273451+0000","last_fullsized":"2024-09-14T04:12:14.273451+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":343,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.338066+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:12:23.338066+0000","last_peered":"2024-09-14T04:12:23.338066+0000","last_clean":"2024-09-14T04:12:23.338066+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:12:23.338066+0000","last_undegraded":"2024-09-14T04:12:23.338066+0000","last_fullsized":"2024-09-14T04:12:23.338066+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":286,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.333411+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:12:23.333411+0000","last_peered":"2024-09-14T04:12:23.333411+0000","last_clean":"2024-09-14T04:12:23.333411+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:12:23.333411+0000","last_undegraded":"2024-09-14T04:12:23.333411+0000","last_fullsized":"2024-09-14T04:12:23.333411+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":354,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.332703+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:12:23.332703+0000","last_peered":"2024-09-14T04:12:23.332703+0000","last_clean":"2024-09-14T04:12:23.332703+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:12:23.332703+0000","last_undegraded":"2024-09-14T04:12:23.332703+0000","last_fullsized":"2024-09-14T04:12:23.332703+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"232'12665","reported_seq":13947,"reported_epoch":232,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:31.278024+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:12:31.278024+0000","last_peered":"2024-09-14T04:12:31.278024+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:12:31.278024+0000","last_undegraded":"2024-09-14T04:12:31.278024+0000","last_fullsized":"2024-09-14T04:12:31.278024+0000","mapping_epoch":210,"log_start":"206'6100","ondisk_log_start":"206'6100","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99549184,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1519,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12665,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1519}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":470,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.295013+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:12:14.295013+0000","last_peered":"2024-09-14T04:12:14.295013+0000","last_clean":"2024-09-14T04:12:14.295013+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:12:14.295013+0000","last_undegraded":"2024-09-14T04:12:14.295013+0000","last_fullsized":"2024-09-14T04:12:14.295013+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":216,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273729+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:12:14.273729+0000","last_peered":"2024-09-14T04:12:14.273729+0000","last_clean":"2024-09-14T04:12:14.273729+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:12:14.273729+0000","last_undegraded":"2024-09-14T04:12:14.273729+0000","last_fullsized":"2024-09-14T04:12:14.273729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":235,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277880+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:12:14.277880+0000","last_peered":"2024-09-14T04:12:14.277880+0000","last_clean":"2024-09-14T04:12:14.277880+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:12:14.277880+0000","last_undegraded":"2024-09-14T04:12:14.277880+0000","last_fullsized":"2024-09-14T04:12:14.277880+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.331852+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:12:23.331852+0000","last_peered":"2024-09-14T04:12:23.331852+0000","last_clean":"2024-09-14T04:12:23.331852+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:12:23.331852+0000","last_undegraded":"2024-09-14T04:12:23.331852+0000","last_fullsized":"2024-09-14T04:12:23.331852+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"232'13002","reported_seq":13060,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:31.134123+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:12:31.134123+0000","last_peered":"2024-09-14T04:12:31.134123+0000","last_clean":"2024-09-14T04:12:31.134123+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:12:31.134123+0000","last_undegraded":"2024-09-14T04:12:31.134123+0000","last_fullsized":"2024-09-14T04:12:31.134123+0000","mapping_epoch":184,"log_start":"199'5100","ondisk_log_start":"199'5100","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13002,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":444,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.267504+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:12:14.267504+0000","last_peered":"2024-09-14T04:12:14.267504+0000","last_clean":"2024-09-14T04:12:14.267504+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:12:14.267504+0000","last_undegraded":"2024-09-14T04:12:14.267504+0000","last_fullsized":"2024-09-14T04:12:14.267504+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":240,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276751+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:12:14.276751+0000","last_peered":"2024-09-14T04:12:14.276751+0000","last_clean":"2024-09-14T04:12:14.276751+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:12:14.276751+0000","last_undegraded":"2024-09-14T04:12:14.276751+0000","last_fullsized":"2024-09-14T04:12:14.276751+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":355,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.339465+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:12:23.339465+0000","last_peered":"2024-09-14T04:12:23.339465+0000","last_clean":"2024-09-14T04:12:23.339465+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:12:23.339465+0000","last_undegraded":"2024-09-14T04:12:23.339465+0000","last_fullsized":"2024-09-14T04:12:23.339465+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":304,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.332006+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:12:23.332006+0000","last_peered":"2024-09-14T04:12:23.332006+0000","last_clean":"2024-09-14T04:12:23.332006+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:12:23.332006+0000","last_undegraded":"2024-09-14T04:12:23.332006+0000","last_fullsized":"2024-09-14T04:12:23.332006+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"232'12804","reported_seq":14002,"reported_epoch":232,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:31.265287+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:12:31.265287+0000","last_peered":"2024-09-14T04:12:31.265287+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:12:31.265287+0000","last_undegraded":"2024-09-14T04:12:31.265287+0000","last_fullsized":"2024-09-14T04:12:31.265287+0000","mapping_epoch":210,"log_start":"206'6200","ondisk_log_start":"206'6200","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714200,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1492,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12804,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":1492}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":478,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.266782+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:12:14.266782+0000","last_peered":"2024-09-14T04:12:14.266782+0000","last_clean":"2024-09-14T04:12:14.266782+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:12:14.266782+0000","last_undegraded":"2024-09-14T04:12:14.266782+0000","last_fullsized":"2024-09-14T04:12:14.266782+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":286,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.339383+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:12:23.339383+0000","last_peered":"2024-09-14T04:12:23.339383+0000","last_clean":"2024-09-14T04:12:23.339383+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:12:23.339383+0000","last_undegraded":"2024-09-14T04:12:23.339383+0000","last_fullsized":"2024-09-14T04:12:23.339383+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":247,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275210+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:12:14.275210+0000","last_peered":"2024-09-14T04:12:14.275210+0000","last_clean":"2024-09-14T04:12:14.275210+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:12:14.275210+0000","last_undegraded":"2024-09-14T04:12:14.275210+0000","last_fullsized":"2024-09-14T04:12:14.275210+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.326169+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:12:23.326169+0000","last_peered":"2024-09-14T04:12:23.326169+0000","last_clean":"2024-09-14T04:12:23.326169+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:12:23.326169+0000","last_undegraded":"2024-09-14T04:12:23.326169+0000","last_fullsized":"2024-09-14T04:12:23.326169+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"232'12919","reported_seq":12977,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:35.812315+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:12:35.812315+0000","last_peered":"2024-09-14T04:12:35.812315+0000","last_clean":"2024-09-14T04:12:35.812315+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:12:35.812315+0000","last_undegraded":"2024-09-14T04:12:35.812315+0000","last_fullsized":"2024-09-14T04:12:35.812315+0000","mapping_epoch":184,"log_start":"205'5512","ondisk_log_start":"205'5512","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12919,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":436,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277663+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:12:14.277663+0000","last_peered":"2024-09-14T04:12:14.277663+0000","last_clean":"2024-09-14T04:12:14.277663+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:12:14.277663+0000","last_undegraded":"2024-09-14T04:12:14.277663+0000","last_fullsized":"2024-09-14T04:12:14.277663+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":165,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.274784+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:12:14.274784+0000","last_peered":"2024-09-14T04:12:14.274784+0000","last_clean":"2024-09-14T04:12:14.274784+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:12:14.274784+0000","last_undegraded":"2024-09-14T04:12:14.274784+0000","last_fullsized":"2024-09-14T04:12:14.274784+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":286,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.330786+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:12:23.330786+0000","last_peered":"2024-09-14T04:12:23.330786+0000","last_clean":"2024-09-14T04:12:23.330786+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:12:23.330786+0000","last_undegraded":"2024-09-14T04:12:23.330786+0000","last_fullsized":"2024-09-14T04:12:23.330786+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":181,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:16.133522+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:12:16.133522+0000","last_peered":"2024-09-14T04:12:16.133522+0000","last_clean":"2024-09-14T04:12:16.133522+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:12:16.133522+0000","last_undegraded":"2024-09-14T04:12:16.133522+0000","last_fullsized":"2024-09-14T04:12:16.133522+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"232'12801","reported_seq":12859,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:31.236248+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:12:31.236248+0000","last_peered":"2024-09-14T04:12:31.236248+0000","last_clean":"2024-09-14T04:12:31.236248+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:12:31.236248+0000","last_undegraded":"2024-09-14T04:12:31.236248+0000","last_fullsized":"2024-09-14T04:12:31.236248+0000","mapping_epoch":184,"log_start":"206'6200","ondisk_log_start":"206'6200","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96403456,"num_objects":1471,"num_object_clones":0,"num_object_copies":4413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12801,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":511,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.338767+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:12:23.338767+0000","last_peered":"2024-09-14T04:12:23.338767+0000","last_clean":"2024-09-14T04:12:23.338767+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:12:23.338767+0000","last_undegraded":"2024-09-14T04:12:23.338767+0000","last_fullsized":"2024-09-14T04:12:23.338767+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":60,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325789+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:12:23.325789+0000","last_peered":"2024-09-14T04:12:23.325789+0000","last_clean":"2024-09-14T04:12:23.325789+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:12:23.325789+0000","last_undegraded":"2024-09-14T04:12:23.325789+0000","last_fullsized":"2024-09-14T04:12:23.325789+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":230,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.267719+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:12:14.267719+0000","last_peered":"2024-09-14T04:12:14.267719+0000","last_clean":"2024-09-14T04:12:14.267719+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:12:14.267719+0000","last_undegraded":"2024-09-14T04:12:14.267719+0000","last_fullsized":"2024-09-14T04:12:14.267719+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":186,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.278134+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:12:14.278134+0000","last_peered":"2024-09-14T04:12:14.278134+0000","last_clean":"2024-09-14T04:12:14.278134+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:12:14.278134+0000","last_undegraded":"2024-09-14T04:12:14.278134+0000","last_fullsized":"2024-09-14T04:12:14.278134+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":208,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277930+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:12:14.277930+0000","last_peered":"2024-09-14T04:12:14.277930+0000","last_clean":"2024-09-14T04:12:14.277930+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:12:14.277930+0000","last_undegraded":"2024-09-14T04:12:14.277930+0000","last_fullsized":"2024-09-14T04:12:14.277930+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":208,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.278653+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:12:14.278653+0000","last_peered":"2024-09-14T04:12:14.278653+0000","last_clean":"2024-09-14T04:12:14.278653+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:12:14.278653+0000","last_undegraded":"2024-09-14T04:12:14.278653+0000","last_fullsized":"2024-09-14T04:12:14.278653+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":286,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.331814+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:12:23.331814+0000","last_peered":"2024-09-14T04:12:23.331814+0000","last_clean":"2024-09-14T04:12:23.331814+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:12:23.331814+0000","last_undegraded":"2024-09-14T04:12:23.331814+0000","last_fullsized":"2024-09-14T04:12:23.331814+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":239,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275020+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:12:14.275020+0000","last_peered":"2024-09-14T04:12:14.275020+0000","last_clean":"2024-09-14T04:12:14.275020+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:12:14.275020+0000","last_undegraded":"2024-09-14T04:12:14.275020+0000","last_fullsized":"2024-09-14T04:12:14.275020+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":200,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273758+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:12:14.273758+0000","last_peered":"2024-09-14T04:12:14.273758+0000","last_clean":"2024-09-14T04:12:14.273758+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:12:14.273758+0000","last_undegraded":"2024-09-14T04:12:14.273758+0000","last_fullsized":"2024-09-14T04:12:14.273758+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":162,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.275298+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:12:14.275298+0000","last_peered":"2024-09-14T04:12:14.275298+0000","last_clean":"2024-09-14T04:12:14.275298+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:12:14.275298+0000","last_undegraded":"2024-09-14T04:12:14.275298+0000","last_fullsized":"2024-09-14T04:12:14.275298+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.328976+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:12:23.328976+0000","last_peered":"2024-09-14T04:12:23.328976+0000","last_clean":"2024-09-14T04:12:23.328976+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:12:23.328976+0000","last_undegraded":"2024-09-14T04:12:23.328976+0000","last_fullsized":"2024-09-14T04:12:23.328976+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":164,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.337726+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:12:23.337726+0000","last_peered":"2024-09-14T04:12:23.337726+0000","last_clean":"2024-09-14T04:12:23.337726+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:12:23.337726+0000","last_undegraded":"2024-09-14T04:12:23.337726+0000","last_fullsized":"2024-09-14T04:12:23.337726+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":199,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.267574+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:12:14.267574+0000","last_peered":"2024-09-14T04:12:14.267574+0000","last_clean":"2024-09-14T04:12:14.267574+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:12:14.267574+0000","last_undegraded":"2024-09-14T04:12:14.267574+0000","last_fullsized":"2024-09-14T04:12:14.267574+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325962+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:12:23.325962+0000","last_peered":"2024-09-14T04:12:23.325962+0000","last_clean":"2024-09-14T04:12:23.325962+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:12:23.325962+0000","last_undegraded":"2024-09-14T04:12:23.325962+0000","last_fullsized":"2024-09-14T04:12:23.325962+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.325566+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:12:23.325566+0000","last_peered":"2024-09-14T04:12:23.325566+0000","last_clean":"2024-09-14T04:12:23.325566+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:12:23.325566+0000","last_undegraded":"2024-09-14T04:12:23.325566+0000","last_fullsized":"2024-09-14T04:12:23.325566+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":313,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.332647+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:12:23.332647+0000","last_peered":"2024-09-14T04:12:23.332647+0000","last_clean":"2024-09-14T04:12:23.332647+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:12:23.332647+0000","last_undegraded":"2024-09-14T04:12:23.332647+0000","last_fullsized":"2024-09-14T04:12:23.332647+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":152,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276526+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:12:14.276526+0000","last_peered":"2024-09-14T04:12:14.276526+0000","last_clean":"2024-09-14T04:12:14.276526+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:12:14.276526+0000","last_undegraded":"2024-09-14T04:12:14.276526+0000","last_fullsized":"2024-09-14T04:12:14.276526+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":210,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.326445+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:12:23.326445+0000","last_peered":"2024-09-14T04:12:23.326445+0000","last_clean":"2024-09-14T04:12:23.326445+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:12:23.326445+0000","last_undegraded":"2024-09-14T04:12:23.326445+0000","last_fullsized":"2024-09-14T04:12:23.326445+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":329,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.329813+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:12:23.329813+0000","last_peered":"2024-09-14T04:12:23.329813+0000","last_clean":"2024-09-14T04:12:23.329813+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:12:23.329813+0000","last_undegraded":"2024-09-14T04:12:23.329813+0000","last_fullsized":"2024-09-14T04:12:23.329813+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":56,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.326067+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:12:23.326067+0000","last_peered":"2024-09-14T04:12:23.326067+0000","last_clean":"2024-09-14T04:12:23.326067+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:12:23.326067+0000","last_undegraded":"2024-09-14T04:12:23.326067+0000","last_fullsized":"2024-09-14T04:12:23.326067+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277514+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:12:14.277514+0000","last_peered":"2024-09-14T04:12:14.277514+0000","last_clean":"2024-09-14T04:12:14.277514+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:12:14.277514+0000","last_undegraded":"2024-09-14T04:12:14.277514+0000","last_fullsized":"2024-09-14T04:12:14.277514+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":238,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.273512+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:12:14.273512+0000","last_peered":"2024-09-14T04:12:14.273512+0000","last_clean":"2024-09-14T04:12:14.273512+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:12:14.273512+0000","last_undegraded":"2024-09-14T04:12:14.273512+0000","last_fullsized":"2024-09-14T04:12:14.273512+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277935+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:12:14.277935+0000","last_peered":"2024-09-14T04:12:14.277935+0000","last_clean":"2024-09-14T04:12:14.277935+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:12:14.277935+0000","last_undegraded":"2024-09-14T04:12:14.277935+0000","last_fullsized":"2024-09-14T04:12:14.277935+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":225,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.276669+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:12:14.276669+0000","last_peered":"2024-09-14T04:12:14.276669+0000","last_clean":"2024-09-14T04:12:14.276669+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:12:14.276669+0000","last_undegraded":"2024-09-14T04:12:14.276669+0000","last_fullsized":"2024-09-14T04:12:14.276669+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.337915+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:12:23.337915+0000","last_peered":"2024-09-14T04:12:23.337915+0000","last_clean":"2024-09-14T04:12:23.337915+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:12:23.337915+0000","last_undegraded":"2024-09-14T04:12:23.337915+0000","last_fullsized":"2024-09-14T04:12:23.337915+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":108,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:16.267702+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:12:16.267702+0000","last_peered":"2024-09-14T04:12:16.267702+0000","last_clean":"2024-09-14T04:12:16.267702+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:12:16.267702+0000","last_undegraded":"2024-09-14T04:12:16.267702+0000","last_fullsized":"2024-09-14T04:12:16.267702+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":199,"reported_epoch":231,"state":"active+clean","last_fresh":"2024-09-14T04:12:14.277422+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:12:14.277422+0000","last_peered":"2024-09-14T04:12:14.277422+0000","last_clean":"2024-09-14T04:12:14.277422+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:12:14.277422+0000","last_undegraded":"2024-09-14T04:12:14.277422+0000","last_fullsized":"2024-09-14T04:12:14.277422+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":62,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-09-14T04:12:23.329102+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:12:23.329102+0000","last_peered":"2024-09-14T04:12:23.329102+0000","last_clean":"2024-09-14T04:12:23.329102+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:12:23.329102+0000","last_undegraded":"2024-09-14T04:12:23.329102+0000","last_fullsized":"2024-09-14T04:12:23.329102+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1533018136,"num_objects":23393,"num_object_clones":0,"num_object_copies":70179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9487,"num_objects_misplaced":10281,"num_objects_unfound":0,"num_objects_dirty":23393,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":206883,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12516,"num_bytes_recovered":760545280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5426982912,"data_stored":5426970696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119940,"ondisk_log_size":119940,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201663,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1295856,"kb_used_data":524776,"kb_used_omap":1,"kb_used_meta":771070,"kb_avail":92449296,"statfs":{"total":95995035648,"available":94668079104,"internally_reserved":0,"allocated":537370624,"data_stored":535897311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":789576139},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463301,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353392,"kb_used_data":700456,"kb_used_omap":1,"kb_used_meta":652926,"kb_avail":92391760,"statfs":{"total":95995035648,"available":94609162240,"internally_reserved":0,"allocated":717266944,"data_stored":715794134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":668596679},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724937,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505700,"kb_used_data":491804,"kb_used_omap":1,"kb_used_meta":1013886,"kb_avail":92239452,"statfs":{"total":95995035648,"available":94453198848,"internally_reserved":0,"allocated":503607296,"data_stored":502142267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1038219718},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986574,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1895088,"kb_used_data":1065640,"kb_used_omap":1,"kb_used_meta":829310,"kb_avail":91850064,"statfs":{"total":95995035648,"available":94054465536,"internally_reserved":0,"allocated":1091215360,"data_stored":1089741762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":849213894},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748602,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994980,"kb_used_data":586716,"kb_used_omap":4,"kb_used_meta":408251,"kb_avail":92750172,"statfs":{"total":95995035648,"available":94976176128,"internally_reserved":0,"allocated":600797184,"data_stored":599331891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":418049580},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912129,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1277604,"kb_used_data":1014364,"kb_used_omap":6,"kb_used_meta":263225,"kb_avail":92467548,"statfs":{"total":95995035648,"available":94686769152,"internally_reserved":0,"allocated":1038708736,"data_stored":1037243945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":269542969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468146,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856152,"kb_used_data":352080,"kb_used_omap":5,"kb_used_meta":503930,"kb_avail":92889000,"statfs":{"total":95995035648,"available":95118336000,"internally_reserved":0,"allocated":360529920,"data_stored":359074856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":516024671},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342261,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":841252,"kb_used_data":643356,"kb_used_omap":10,"kb_used_meta":197877,"kb_avail":92903900,"statfs":{"total":95995035648,"available":95133593600,"internally_reserved":0,"allocated":658796544,"data_stored":657331231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":202626258},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":652541952,"data_stored":652541952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":354156544,"data_stored":354156544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1025048576,"data_stored":1025048576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":595398656,"data_stored":595394584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1079316480,"data_stored":1079312408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":495124480,"data_stored":495124480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":694554624,"data_stored":694550552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":530841600,"data_stored":530841600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:12:42.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:12:42.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:42.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:42.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:42.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:42.347 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:42 smithi001 ceph-mon[107399]: pgmap v807: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9076/57423 objects degraded (15.806%); 9766/57423 objects misplaced (17.007%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:42.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:42 smithi001 ceph-mon[112165]: pgmap v807: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9076/57423 objects degraded (15.806%); 9766/57423 objects misplaced (17.007%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:42.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:42.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:42.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:42 smithi012 ceph-mon[101180]: pgmap v807: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9076/57423 objects degraded (15.806%); 9766/57423 objects misplaced (17.007%); 0 B/s, 76 objects/s recovering 2024-09-14T04:12:42.605 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:42.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:42.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:42.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:42.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:42.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:42.952 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:12:42.952 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:12:42.952 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:12:42.952 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:12:42.952 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:12:42.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:43.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:43.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:43.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:43.261 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:43.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:43.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:43.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:43.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:43.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:43.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:43.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:43.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:12:43.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:43.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:43.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:43.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:44.007 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:44.109 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:44.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:44.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:44.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:44.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:44.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:44.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:44.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:44 smithi001 ceph-mon[112165]: pgmap v808: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 8555/50685 objects degraded (16.879%); 9299/50685 objects misplaced (18.347%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8555/50685 objects degraded (16.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:44 smithi001 ceph-mon[107399]: pgmap v808: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 8555/50685 objects degraded (16.879%); 9299/50685 objects misplaced (18.347%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8555/50685 objects degraded (16.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:44.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:44 smithi012 ceph-mon[101180]: pgmap v808: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 8555/50685 objects degraded (16.879%); 9299/50685 objects misplaced (18.347%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8555/50685 objects degraded (16.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:44.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:44.629 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:44.674 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:44.699 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:44.709 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:44.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:44.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:44.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:44.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:44.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:45.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:45.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:45.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:45.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:45.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:45.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:45.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:45.568 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:45.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:45.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:45.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:45.825 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:45.848 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:45.950 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:45.976 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:45.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:46.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:46.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:46.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:46.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:12:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:46.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:46.329 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:46.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:46.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:46.466 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:46 smithi001 ceph-mon[107399]: pgmap v809: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1010 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8555/48543 objects degraded (17.624%); 9299/48543 objects misplaced (19.156%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:46.466 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:46 smithi001 ceph-mon[107399]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T04:12:46.466 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:46 smithi001 ceph-mon[112165]: pgmap v809: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1010 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8555/48543 objects degraded (17.624%); 9299/48543 objects misplaced (19.156%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:46.467 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:46 smithi001 ceph-mon[112165]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T04:12:46.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:46 smithi012 ceph-mon[101180]: pgmap v809: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 93 active+clean; 1010 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8555/48543 objects degraded (17.624%); 9299/48543 objects misplaced (19.156%); 0 B/s, 93 objects/s recovering 2024-09-14T04:12:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:46 smithi012 ceph-mon[101180]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T04:12:46.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:46.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:46.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:46] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-09-14T04:12:46.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:46.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:46.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:46.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:46.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:47.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:47.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:47.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:47.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:47.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:47.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:47.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:47.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:47.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:47.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:47.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:47 smithi012 ceph-mon[101180]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T04:12:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:47.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:47 smithi001 ceph-mon[107399]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T04:12:47.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:47 smithi001 ceph-mon[112165]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T04:12:47.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:47.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:47.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:47.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:48.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:48.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:48.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:48.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:48.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:48.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:48.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:48.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:48.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[107399]: pgmap v812: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 876 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 5257/42105 objects degraded (12.485%); 9674/42105 objects misplaced (22.976%); 1008 KiB/s, 134 objects/s recovering 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[112165]: pgmap v812: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 876 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 5257/42105 objects degraded (12.485%); 9674/42105 objects misplaced (22.976%); 1008 KiB/s, 134 objects/s recovering 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.536 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:48.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:48.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:48.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:48 smithi012 ceph-mon[101180]: pgmap v812: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 876 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 5257/42105 objects degraded (12.485%); 9674/42105 objects misplaced (22.976%); 1008 KiB/s, 134 objects/s recovering 2024-09-14T04:12:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:48.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:48.707 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:48.737 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:48.737 INFO:teuthology.orchestra.run.smithi001.stdout:{"fsid":"5626d550-724a-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5257/42105 objects degraded (12.485%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1190,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":235,"num_osds":8,"num_up_osds":8,"osd_up_since":1726286894,"num_in_osds":8,"osd_in_since":1726286805,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":94},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":14035,"data_bytes":918624253,"bytes_used":8484204544,"bytes_avail":759476080640,"bytes_total":767960285184,"degraded_objects":5257,"degraded_total":42105,"degraded_ratio":0.12485453034081463,"misplaced_objects":9674,"misplaced_total":42105,"misplaced_ratio":0.22975893599334996,"recovering_objects_per_sec":134,"recovering_bytes_per_sec":1031873,"recovering_keys_per_sec":0,"num_objects_recovered":1073,"num_bytes_recovered":8257536,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":788},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi001.ytjgbj","status":"up:active","gid":24557}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-14T04:09:15.106156+0000","services":{}},"progress_events":{}} 2024-09-14T04:12:48.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:48.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:48.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:48.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:48.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:49.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:49.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:49.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:49.310 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:49.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:49.432 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1913641557' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:49.433 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5257/42105 objects degraded (12.485%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:49.433 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1913641557' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:49.433 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5257/42105 objects degraded (12.485%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:49.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:49.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:49.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1913641557' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:12:49.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5257/42105 objects degraded (12.485%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:49.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:49.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:49.837 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:49.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:49.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:49.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:49.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:50.010 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:12:50.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:12:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:50.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:50.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:50.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:50.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:50.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.433 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:50.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:50.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[107399]: pgmap v813: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 822 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 5116/39522 objects degraded (12.945%); 9674/39522 objects misplaced (24.478%); 1008 KiB/s, 151 objects/s recovering 2024-09-14T04:12:50.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:50.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:50.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[112165]: pgmap v813: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 822 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 5116/39522 objects degraded (12.945%); 9674/39522 objects misplaced (24.478%); 1008 KiB/s, 151 objects/s recovering 2024-09-14T04:12:50.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:50.494 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:50.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:50.567 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:50 smithi012 ceph-mon[101180]: pgmap v813: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 822 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 5116/39522 objects degraded (12.945%); 9674/39522 objects misplaced (24.478%); 1008 KiB/s, 151 objects/s recovering 2024-09-14T04:12:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:50.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:50.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:50.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:50.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:50.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:50.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:50.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:50.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:51.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:51.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:51.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:51.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:51.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:51.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:51.418 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:12:51.419 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:12:51.455 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:12:51.455 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:12:51.490 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:12:51.490 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:12:51.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:51.523 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:51.524 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:51.524 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:51.524 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:12:51.524 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:12:51.525 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:51.525 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:51.525 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:51.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:51.557 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:12:51.557 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:12:51.589 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:12:51.589 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:12:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:12:51.621 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:12:51.621 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:12:51.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:51.652 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:12:51.653 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:12:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:51.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:51.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:51.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:51.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:51.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:52.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:52.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:52.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:52.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:52.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:52.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:52.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:52.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:52.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:52 smithi001 ceph-mon[112165]: pgmap v814: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 755 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5116/36297 objects degraded (14.095%); 9674/36297 objects misplaced (26.652%); 1008 KiB/s, 86 objects/s recovering 2024-09-14T04:12:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/42473140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:52 smithi001 ceph-mon[107399]: pgmap v814: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 755 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5116/36297 objects degraded (14.095%); 9674/36297 objects misplaced (26.652%); 1008 KiB/s, 86 objects/s recovering 2024-09-14T04:12:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/42473140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:52.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:52 smithi012 ceph-mon[101180]: pgmap v814: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 755 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5116/36297 objects degraded (14.095%); 9674/36297 objects misplaced (26.652%); 1008 KiB/s, 86 objects/s recovering 2024-09-14T04:12:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/42473140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:12:52.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:52.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:52.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:52.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:52.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:52.898 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:52.936 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:52.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:52.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:53.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:53.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:53.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:53.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:53.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:53.338 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:12:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:53.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:53.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:53.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:53.511 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:12:53.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:53.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:53.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:53.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:53.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:53.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:53.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:54.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:54.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:54.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:54.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:54.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.380 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:54.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:54.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.518 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:54.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:54 smithi012 ceph-mon[101180]: pgmap v815: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 606 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 4971/29148 objects degraded (17.054%); 8968/29148 objects misplaced (30.767%); 2.8 MiB/s, 134 objects/s recovering 2024-09-14T04:12:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:54 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:54 smithi012 ceph-mon[101180]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T04:12:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4971/29148 objects degraded (17.054%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:54.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:54.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[107399]: pgmap v815: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 606 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 4971/29148 objects degraded (17.054%); 8968/29148 objects misplaced (30.767%); 2.8 MiB/s, 134 objects/s recovering 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[107399]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4971/29148 objects degraded (17.054%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[112165]: pgmap v815: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 606 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 4971/29148 objects degraded (17.054%); 8968/29148 objects misplaced (30.767%); 2.8 MiB/s, 134 objects/s recovering 2024-09-14T04:12:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:12:54.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4015220593' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:12:54.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[112165]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T04:12:54.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4971/29148 objects degraded (17.054%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:54.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:54.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:54.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:54.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:54.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:55.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:55.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:55.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:55.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:55.283 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:55.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:55.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:12:55.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:55.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:55.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:55.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:55.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:55.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:55.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:56.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:56.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:56.240 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:56.280 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:56.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:12:56.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:56.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:56 smithi001 ceph-mon[112165]: pgmap v817: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 560 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 4971/26913 objects degraded (18.471%); 8968/26913 objects misplaced (33.322%); 2.5 MiB/s, 120 objects/s recovering 2024-09-14T04:12:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:56 smithi001 ceph-mon[107399]: pgmap v817: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 560 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 4971/26913 objects degraded (18.471%); 8968/26913 objects misplaced (33.322%); 2.5 MiB/s, 120 objects/s recovering 2024-09-14T04:12:56.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:56.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:56 smithi012 ceph-mon[101180]: pgmap v817: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 560 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 4971/26913 objects degraded (18.471%); 8968/26913 objects misplaced (33.322%); 2.5 MiB/s, 120 objects/s recovering 2024-09-14T04:12:56.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.725 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:56.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:56.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:56.830 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:12:56.830 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:12:56.833 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:12:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:12:56] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-14T04:12:56.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:56.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:56.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:56.965 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:12:56.998 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:12:57.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:57.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:57.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:57.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:57.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:57.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:57.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:57.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:57.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:57.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:57.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:57.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:57.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:57.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:58.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:58.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:58.067 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:58.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:58.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:12:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:58.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:58.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:58.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:58.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:58 smithi001 ceph-mon[107399]: pgmap v818: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 409 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4971/19665 objects degraded (25.278%); 8246/19665 objects misplaced (41.932%); 3.1 MiB/s, 77 objects/s recovering 2024-09-14T04:12:58.387 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:58 smithi001 ceph-mon[112165]: pgmap v818: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 409 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4971/19665 objects degraded (25.278%); 8246/19665 objects misplaced (41.932%); 3.1 MiB/s, 77 objects/s recovering 2024-09-14T04:12:58.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:58.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:58.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:12:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:58.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:12:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:58 smithi012 ceph-mon[101180]: pgmap v818: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 409 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4971/19665 objects degraded (25.278%); 8246/19665 objects misplaced (41.932%); 3.1 MiB/s, 77 objects/s recovering 2024-09-14T04:12:58.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:58.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:58.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:58.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:12:58.865 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:58.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:12:58.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:12:59.004 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:12:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:59.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:12:59.106 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:59.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:59.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:59.310 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:12:59.312 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:12:59.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:59.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:12:59.461 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:12:59.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:12:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:12:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4971/19665 objects degraded (25.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:59.603 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:59.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:12:59.705 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:12:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:12:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4971/19665 objects degraded (25.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:12:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4971/19665 objects degraded (25.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:12:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:12:59.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:12:59.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:12:59.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:12:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:12:59.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:12:59.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:00.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:00.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:00.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:00.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:00.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:00.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:00.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:00 smithi012 ceph-mon[101180]: pgmap v819: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 348 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4800/16737 objects degraded (28.679%); 8246/16737 objects misplaced (49.268%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:00 smithi012 ceph-mon[101180]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:00.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:00.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:00 smithi001 ceph-mon[107399]: pgmap v819: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 348 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4800/16737 objects degraded (28.679%); 8246/16737 objects misplaced (49.268%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:00 smithi001 ceph-mon[107399]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:00 smithi001 ceph-mon[112165]: pgmap v819: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 348 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4800/16737 objects degraded (28.679%); 8246/16737 objects misplaced (49.268%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:00 smithi001 ceph-mon[112165]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:00.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:00.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:00.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:00.984 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:00.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:01.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:01.125 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:01.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:01.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:01.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:01.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:01.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:01.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:01.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:01.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:01.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:01.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:01.919 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":819,"stamp":"2024-09-14T04:12:59.199216+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":364451837,"num_objects":5579,"num_object_clones":0,"num_object_copies":16737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4800,"num_objects_misplaced":8246,"num_objects_unfound":0,"num_objects_dirty":5579,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224787,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14987,"num_bytes_recovered":801965522,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121678,"ondisk_log_size":121678,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6692664,"kb_used_data":2041720,"kb_used_omap":33,"kb_used_meta":4650526,"kb_avail":743268552,"statfs":{"total":767960285184,"available":761106997248,"internally_reserved":0,"allocated":2090721280,"data_stored":2078737221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4762139060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-497745920,"num_objects":-7595,"num_object_clones":0,"num_object_copies":-22785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-316,"num_objects_misplaced":-1428,"num_objects_unfound":0,"num_objects_dirty":-7595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7595,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":808,"num_bytes_recovered":32243712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005797"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":262,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.373175+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:12:53.373175+0000","last_peered":"2024-09-14T04:12:53.373175+0000","last_clean":"2024-09-14T04:12:53.373175+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:12:53.373175+0000","last_undegraded":"2024-09-14T04:12:53.373175+0000","last_fullsized":"2024-09-14T04:12:53.373175+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":215,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349413+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:12:53.349413+0000","last_peered":"2024-09-14T04:12:53.349413+0000","last_clean":"2024-09-14T04:12:53.349413+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:12:53.349413+0000","last_undegraded":"2024-09-14T04:12:53.349413+0000","last_fullsized":"2024-09-14T04:12:53.349413+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":46,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269036+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:12:46.269036+0000","last_peered":"2024-09-14T04:12:46.269036+0000","last_clean":"2024-09-14T04:12:46.269036+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:12:46.269036+0000","last_undegraded":"2024-09-14T04:12:46.269036+0000","last_fullsized":"2024-09-14T04:12:46.269036+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347637+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:12:53.347637+0000","last_peered":"2024-09-14T04:12:53.347637+0000","last_clean":"2024-09-14T04:12:53.347637+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:12:53.347637+0000","last_undegraded":"2024-09-14T04:12:53.347637+0000","last_fullsized":"2024-09-14T04:12:53.347637+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347917+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:12:53.347917+0000","last_peered":"2024-09-14T04:12:53.347917+0000","last_clean":"2024-09-14T04:12:53.347917+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:12:53.347917+0000","last_undegraded":"2024-09-14T04:12:53.347917+0000","last_fullsized":"2024-09-14T04:12:53.347917+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.340446+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:12:53.340446+0000","last_peered":"2024-09-14T04:12:53.340446+0000","last_clean":"2024-09-14T04:12:53.340446+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:12:53.340446+0000","last_undegraded":"2024-09-14T04:12:53.340446+0000","last_fullsized":"2024-09-14T04:12:53.340446+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":172,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.356932+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:12:53.356932+0000","last_peered":"2024-09-14T04:12:53.356932+0000","last_clean":"2024-09-14T04:12:53.356932+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:12:53.356932+0000","last_undegraded":"2024-09-14T04:12:53.356932+0000","last_fullsized":"2024-09-14T04:12:53.356932+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":287,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.352536+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:12:53.352536+0000","last_peered":"2024-09-14T04:12:53.352536+0000","last_clean":"2024-09-14T04:12:53.352536+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:12:53.352536+0000","last_undegraded":"2024-09-14T04:12:53.352536+0000","last_fullsized":"2024-09-14T04:12:53.352536+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"236'14461","reported_seq":14523,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:56.844882+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:12:56.844882+0000","last_peered":"2024-09-14T04:12:56.844882+0000","last_clean":"2024-09-14T04:12:56.844882+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:12:56.844882+0000","last_undegraded":"2024-09-14T04:12:56.844882+0000","last_fullsized":"2024-09-14T04:12:56.844882+0000","mapping_epoch":184,"log_start":"199'4700","ondisk_log_start":"199'4700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9761,"log_dups_size":0,"ondisk_log_size":9761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23134208,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14461,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":247,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.336018+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:12:46.336018+0000","last_peered":"2024-09-14T04:12:46.336018+0000","last_clean":"2024-09-14T04:12:46.336018+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:12:46.336018+0000","last_undegraded":"2024-09-14T04:12:46.336018+0000","last_fullsized":"2024-09-14T04:12:46.336018+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":46,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268342+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:12:46.268342+0000","last_peered":"2024-09-14T04:12:46.268342+0000","last_clean":"2024-09-14T04:12:46.268342+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:12:46.268342+0000","last_undegraded":"2024-09-14T04:12:46.268342+0000","last_fullsized":"2024-09-14T04:12:46.268342+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"236'14209","reported_seq":26352,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:56.292034+0000","last_change":"2024-09-14T04:12:46.311123+0000","last_active":"2024-09-14T04:12:56.292034+0000","last_peered":"2024-09-14T04:12:56.292034+0000","last_clean":"2024-09-14T04:12:56.292034+0000","last_became_active":"2024-09-14T04:12:46.274363+0000","last_became_peered":"2024-09-14T04:12:46.274363+0000","last_unstale":"2024-09-14T04:12:56.292034+0000","last_undegraded":"2024-09-14T04:12:56.292034+0000","last_fullsized":"2024-09-14T04:12:56.292034+0000","mapping_epoch":234,"log_start":"208'7600","ondisk_log_start":"208'7600","created":184,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:52:16.466037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23789568,"num_objects":363,"num_object_clones":0,"num_object_copies":1089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14209,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6022,"num_bytes_recovered":272302080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":246,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.336110+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:12:46.336110+0000","last_peered":"2024-09-14T04:12:46.336110+0000","last_clean":"2024-09-14T04:12:46.336110+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:12:46.336110+0000","last_undegraded":"2024-09-14T04:12:46.336110+0000","last_fullsized":"2024-09-14T04:12:46.336110+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":237,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268816+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:12:46.268816+0000","last_peered":"2024-09-14T04:12:46.268816+0000","last_clean":"2024-09-14T04:12:46.268816+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:12:46.268816+0000","last_undegraded":"2024-09-14T04:12:46.268816+0000","last_fullsized":"2024-09-14T04:12:46.268816+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"236'14114","reported_seq":15416,"reported_epoch":236,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:12:54.912734+0000","last_change":"2024-09-14T04:10:36.624348+0000","last_active":"2024-09-14T04:12:54.912734+0000","last_peered":"2024-09-14T04:12:54.912734+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:12:54.912734+0000","last_undegraded":"2024-09-14T04:12:54.912734+0000","last_fullsized":"2024-09-14T04:12:54.912734+0000","mapping_epoch":210,"log_start":"206'6200","ondisk_log_start":"206'6200","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7270,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14114,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":594,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-6435}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":217,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.336110+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:12:46.336110+0000","last_peered":"2024-09-14T04:12:46.336110+0000","last_clean":"2024-09-14T04:12:46.336110+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:12:46.336110+0000","last_undegraded":"2024-09-14T04:12:46.336110+0000","last_fullsized":"2024-09-14T04:12:46.336110+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.271133+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:12:46.271133+0000","last_peered":"2024-09-14T04:12:46.271133+0000","last_clean":"2024-09-14T04:12:46.271133+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:12:46.271133+0000","last_undegraded":"2024-09-14T04:12:46.271133+0000","last_fullsized":"2024-09-14T04:12:46.271133+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"236'14231","reported_seq":14348,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:57.719173+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:12:57.719173+0000","last_peered":"2024-09-14T04:12:57.719173+0000","last_clean":"2024-09-14T04:12:57.719173+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:12:57.719173+0000","last_undegraded":"2024-09-14T04:12:57.719173+0000","last_fullsized":"2024-09-14T04:12:57.719173+0000","mapping_epoch":184,"log_start":"205'5800","ondisk_log_start":"205'5800","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18677760,"num_objects":285,"num_object_clones":0,"num_object_copies":855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14231,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":223,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268928+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:12:46.268928+0000","last_peered":"2024-09-14T04:12:46.268928+0000","last_clean":"2024-09-14T04:12:46.268928+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:12:46.268928+0000","last_undegraded":"2024-09-14T04:12:46.268928+0000","last_fullsized":"2024-09-14T04:12:46.268928+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":65,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349476+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:12:53.349476+0000","last_peered":"2024-09-14T04:12:53.349476+0000","last_clean":"2024-09-14T04:12:53.349476+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:12:53.349476+0000","last_undegraded":"2024-09-14T04:12:53.349476+0000","last_fullsized":"2024-09-14T04:12:53.349476+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"236'13747","reported_seq":20633,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:54.958118+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:12:54.958118+0000","last_peered":"2024-09-14T04:12:54.958118+0000","last_clean":"2024-09-14T04:12:54.958118+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:12:54.958118+0000","last_undegraded":"2024-09-14T04:12:54.958118+0000","last_fullsized":"2024-09-14T04:12:54.958118+0000","mapping_epoch":217,"log_start":"206'5800","ondisk_log_start":"206'5800","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7947,"log_dups_size":0,"ondisk_log_size":7947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28508160,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13747,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":588,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.339224+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:12:46.339224+0000","last_peered":"2024-09-14T04:12:46.339224+0000","last_clean":"2024-09-14T04:12:46.339224+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:12:46.339224+0000","last_undegraded":"2024-09-14T04:12:46.339224+0000","last_fullsized":"2024-09-14T04:12:46.339224+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":69,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349242+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:12:53.349242+0000","last_peered":"2024-09-14T04:12:53.349242+0000","last_clean":"2024-09-14T04:12:53.349242+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:12:53.349242+0000","last_undegraded":"2024-09-14T04:12:53.349242+0000","last_fullsized":"2024-09-14T04:12:53.349242+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":222,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269372+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:12:46.269372+0000","last_peered":"2024-09-14T04:12:46.269372+0000","last_clean":"2024-09-14T04:12:46.269372+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:12:46.269372+0000","last_undegraded":"2024-09-14T04:12:46.269372+0000","last_fullsized":"2024-09-14T04:12:46.269372+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"236'14057","reported_seq":14174,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:57.717544+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:12:57.717544+0000","last_peered":"2024-09-14T04:12:57.717544+0000","last_clean":"2024-09-14T04:12:57.717544+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:12:57.717544+0000","last_undegraded":"2024-09-14T04:12:57.717544+0000","last_fullsized":"2024-09-14T04:12:57.717544+0000","mapping_epoch":184,"log_start":"205'5700","ondisk_log_start":"205'5700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8357,"log_dups_size":0,"ondisk_log_size":8357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18022400,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14057,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":457,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268279+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:12:46.268279+0000","last_peered":"2024-09-14T04:12:46.268279+0000","last_clean":"2024-09-14T04:12:46.268279+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:12:46.268279+0000","last_undegraded":"2024-09-14T04:12:46.268279+0000","last_fullsized":"2024-09-14T04:12:46.268279+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.271284+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:12:46.271284+0000","last_peered":"2024-09-14T04:12:46.271284+0000","last_clean":"2024-09-14T04:12:46.271284+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:12:46.271284+0000","last_undegraded":"2024-09-14T04:12:46.271284+0000","last_fullsized":"2024-09-14T04:12:46.271284+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":26,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269435+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:12:46.269435+0000","last_peered":"2024-09-14T04:12:46.269435+0000","last_clean":"2024-09-14T04:12:46.269435+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:12:46.269435+0000","last_undegraded":"2024-09-14T04:12:46.269435+0000","last_fullsized":"2024-09-14T04:12:46.269435+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"236'13907","reported_seq":17459,"reported_epoch":236,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:56.396080+0000","last_change":"2024-09-14T04:12:44.596502+0000","last_active":"2024-09-14T04:12:56.396080+0000","last_peered":"2024-09-14T04:12:56.396080+0000","last_clean":"2024-09-14T04:09:49.209154+0000","last_became_active":"2024-09-14T04:11:13.358760+0000","last_became_peered":"2024-09-14T04:11:13.358760+0000","last_unstale":"2024-09-14T04:12:56.396080+0000","last_undegraded":"2024-09-14T04:11:13.259138+0000","last_fullsized":"2024-09-14T04:11:13.243084+0000","mapping_epoch":220,"log_start":"208'7300","ondisk_log_start":"208'7300","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:46.130184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23920640,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":965,"num_objects_misplaced":275,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13907,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1735,"num_bytes_recovered":113704960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":-875},{"shards":"1,6,7","objects":275}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":26,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268511+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:12:46.268511+0000","last_peered":"2024-09-14T04:12:46.268511+0000","last_clean":"2024-09-14T04:12:46.268511+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:12:46.268511+0000","last_undegraded":"2024-09-14T04:12:46.268511+0000","last_fullsized":"2024-09-14T04:12:46.268511+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":204,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269785+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:12:46.269785+0000","last_peered":"2024-09-14T04:12:46.269785+0000","last_clean":"2024-09-14T04:12:46.269785+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:12:46.269785+0000","last_undegraded":"2024-09-14T04:12:46.269785+0000","last_fullsized":"2024-09-14T04:12:46.269785+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"236'14036","reported_seq":14098,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:55.861239+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:12:55.861239+0000","last_peered":"2024-09-14T04:12:55.861239+0000","last_clean":"2024-09-14T04:12:55.861239+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:12:55.861239+0000","last_undegraded":"2024-09-14T04:12:55.861239+0000","last_fullsized":"2024-09-14T04:12:55.861239+0000","mapping_epoch":184,"log_start":"207'6672","ondisk_log_start":"207'6672","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7364,"log_dups_size":0,"ondisk_log_size":7364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14036,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":441,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:51.922823+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:12:51.922823+0000","last_peered":"2024-09-14T04:12:51.922823+0000","last_clean":"2024-09-14T04:12:51.922823+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:12:51.922823+0000","last_undegraded":"2024-09-14T04:12:51.922823+0000","last_fullsized":"2024-09-14T04:12:51.922823+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":172,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347306+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:12:53.347306+0000","last_peered":"2024-09-14T04:12:53.347306+0000","last_clean":"2024-09-14T04:12:53.347306+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:12:53.347306+0000","last_undegraded":"2024-09-14T04:12:53.347306+0000","last_fullsized":"2024-09-14T04:12:53.347306+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":308,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.369630+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:12:53.369630+0000","last_peered":"2024-09-14T04:12:53.369630+0000","last_clean":"2024-09-14T04:12:53.369630+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:12:53.369630+0000","last_undegraded":"2024-09-14T04:12:53.369630+0000","last_fullsized":"2024-09-14T04:12:53.369630+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"236'14134","reported_seq":14194,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:56.405548+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:12:56.405548+0000","last_peered":"2024-09-14T04:12:56.405548+0000","last_clean":"2024-09-14T04:12:56.405548+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:12:56.405548+0000","last_undegraded":"2024-09-14T04:12:56.405548+0000","last_fullsized":"2024-09-14T04:12:56.405548+0000","mapping_epoch":184,"log_start":"208'7600","ondisk_log_start":"208'7600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14134,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":605,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335841+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:12:46.335841+0000","last_peered":"2024-09-14T04:12:46.335841+0000","last_clean":"2024-09-14T04:12:46.335841+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:12:46.335841+0000","last_undegraded":"2024-09-14T04:12:46.335841+0000","last_fullsized":"2024-09-14T04:12:46.335841+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":220,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268575+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:12:46.268575+0000","last_peered":"2024-09-14T04:12:46.268575+0000","last_clean":"2024-09-14T04:12:46.268575+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:12:46.268575+0000","last_undegraded":"2024-09-14T04:12:46.268575+0000","last_fullsized":"2024-09-14T04:12:46.268575+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":167,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269287+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:12:46.269287+0000","last_peered":"2024-09-14T04:12:46.269287+0000","last_clean":"2024-09-14T04:12:46.269287+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:12:46.269287+0000","last_undegraded":"2024-09-14T04:12:46.269287+0000","last_fullsized":"2024-09-14T04:12:46.269287+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.271802+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:12:46.271802+0000","last_peered":"2024-09-14T04:12:46.271802+0000","last_clean":"2024-09-14T04:12:46.271802+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:12:46.271802+0000","last_undegraded":"2024-09-14T04:12:46.271802+0000","last_fullsized":"2024-09-14T04:12:46.271802+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"236'13899","reported_seq":14015,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:58.838312+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:12:58.838312+0000","last_peered":"2024-09-14T04:12:58.838312+0000","last_clean":"2024-09-14T04:12:58.838312+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:12:58.838312+0000","last_undegraded":"2024-09-14T04:12:58.838312+0000","last_fullsized":"2024-09-14T04:12:58.838312+0000","mapping_epoch":184,"log_start":"208'6859","ondisk_log_start":"208'6859","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15400960,"num_objects":235,"num_object_clones":0,"num_object_copies":705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13899,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":492,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.367604+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:12:53.367604+0000","last_peered":"2024-09-14T04:12:53.367604+0000","last_clean":"2024-09-14T04:12:53.367604+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:12:53.367604+0000","last_undegraded":"2024-09-14T04:12:53.367604+0000","last_fullsized":"2024-09-14T04:12:53.367604+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":249,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335759+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:12:46.335759+0000","last_peered":"2024-09-14T04:12:46.335759+0000","last_clean":"2024-09-14T04:12:46.335759+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:12:46.335759+0000","last_undegraded":"2024-09-14T04:12:46.335759+0000","last_fullsized":"2024-09-14T04:12:46.335759+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":288,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.348331+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:12:53.348331+0000","last_peered":"2024-09-14T04:12:53.348331+0000","last_clean":"2024-09-14T04:12:53.348331+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:12:53.348331+0000","last_undegraded":"2024-09-14T04:12:53.348331+0000","last_fullsized":"2024-09-14T04:12:53.348331+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.348001+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:12:53.348001+0000","last_peered":"2024-09-14T04:12:53.348001+0000","last_clean":"2024-09-14T04:12:53.348001+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:12:53.348001+0000","last_undegraded":"2024-09-14T04:12:53.348001+0000","last_fullsized":"2024-09-14T04:12:53.348001+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"236'14117","reported_seq":18377,"reported_epoch":236,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:12:57.736192+0000","last_change":"2024-09-14T04:12:44.608526+0000","last_active":"2024-09-14T04:12:57.736192+0000","last_peered":"2024-09-14T04:12:57.736192+0000","last_clean":"2024-09-14T04:09:49.197821+0000","last_became_active":"2024-09-14T04:09:51.414801+0000","last_became_peered":"2024-09-14T04:09:51.414801+0000","last_unstale":"2024-09-14T04:12:57.736192+0000","last_undegraded":"2024-09-14T04:10:35.401112+0000","last_fullsized":"2024-09-14T04:09:51.215769+0000","mapping_epoch":202,"log_start":"205'5700","ondisk_log_start":"205'5700","created":184,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3835,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14117,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2067,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,7","objects":-3518}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":468,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335574+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:12:46.335574+0000","last_peered":"2024-09-14T04:12:46.335574+0000","last_clean":"2024-09-14T04:12:46.335574+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:12:46.335574+0000","last_undegraded":"2024-09-14T04:12:46.335574+0000","last_fullsized":"2024-09-14T04:12:46.335574+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":351,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.379157+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:12:53.379157+0000","last_peered":"2024-09-14T04:12:53.379157+0000","last_clean":"2024-09-14T04:12:53.379157+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:12:53.379157+0000","last_undegraded":"2024-09-14T04:12:53.379157+0000","last_fullsized":"2024-09-14T04:12:53.379157+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":294,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.368903+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:12:53.368903+0000","last_peered":"2024-09-14T04:12:53.368903+0000","last_clean":"2024-09-14T04:12:53.368903+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:12:53.368903+0000","last_undegraded":"2024-09-14T04:12:53.368903+0000","last_fullsized":"2024-09-14T04:12:53.368903+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":362,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.369456+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:12:53.369456+0000","last_peered":"2024-09-14T04:12:53.369456+0000","last_clean":"2024-09-14T04:12:53.369456+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:12:53.369456+0000","last_undegraded":"2024-09-14T04:12:53.369456+0000","last_fullsized":"2024-09-14T04:12:53.369456+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"236'13814","reported_seq":15100,"reported_epoch":236,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:56.360118+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:12:56.360118+0000","last_peered":"2024-09-14T04:12:56.360118+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:12:56.360118+0000","last_undegraded":"2024-09-14T04:12:56.360118+0000","last_fullsized":"2024-09-14T04:12:56.360118+0000","mapping_epoch":210,"log_start":"208'7200","ondisk_log_start":"208'7200","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24248320,"num_objects":370,"num_object_clones":0,"num_object_copies":1110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":370,"num_objects_unfound":0,"num_objects_dirty":370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13814,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":370}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":475,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268556+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:12:46.268556+0000","last_peered":"2024-09-14T04:12:46.268556+0000","last_clean":"2024-09-14T04:12:46.268556+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:12:46.268556+0000","last_undegraded":"2024-09-14T04:12:46.268556+0000","last_fullsized":"2024-09-14T04:12:46.268556+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":221,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269423+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:12:46.269423+0000","last_peered":"2024-09-14T04:12:46.269423+0000","last_clean":"2024-09-14T04:12:46.269423+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:12:46.269423+0000","last_undegraded":"2024-09-14T04:12:46.269423+0000","last_fullsized":"2024-09-14T04:12:46.269423+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":239,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269147+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:12:46.269147+0000","last_peered":"2024-09-14T04:12:46.269147+0000","last_clean":"2024-09-14T04:12:46.269147+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:12:46.269147+0000","last_undegraded":"2024-09-14T04:12:46.269147+0000","last_fullsized":"2024-09-14T04:12:46.269147+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":235,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268690+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:12:46.268690+0000","last_peered":"2024-09-14T04:12:46.268690+0000","last_clean":"2024-09-14T04:12:46.268690+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:12:46.268690+0000","last_undegraded":"2024-09-14T04:12:46.268690+0000","last_fullsized":"2024-09-14T04:12:46.268690+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"236'14253","reported_seq":14315,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:56.143661+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:12:56.143661+0000","last_peered":"2024-09-14T04:12:56.143661+0000","last_clean":"2024-09-14T04:12:56.143661+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:12:56.143661+0000","last_undegraded":"2024-09-14T04:12:56.143661+0000","last_fullsized":"2024-09-14T04:12:56.143661+0000","mapping_epoch":184,"log_start":"206'6300","ondisk_log_start":"206'6300","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22216704,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14253,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":449,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:51.415478+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:12:51.415478+0000","last_peered":"2024-09-14T04:12:51.415478+0000","last_clean":"2024-09-14T04:12:51.415478+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:12:51.415478+0000","last_undegraded":"2024-09-14T04:12:51.415478+0000","last_fullsized":"2024-09-14T04:12:51.415478+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":244,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269156+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:12:46.269156+0000","last_peered":"2024-09-14T04:12:46.269156+0000","last_clean":"2024-09-14T04:12:46.269156+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:12:46.269156+0000","last_undegraded":"2024-09-14T04:12:46.269156+0000","last_fullsized":"2024-09-14T04:12:46.269156+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":363,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.378955+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:12:53.378955+0000","last_peered":"2024-09-14T04:12:53.378955+0000","last_clean":"2024-09-14T04:12:53.378955+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:12:53.378955+0000","last_undegraded":"2024-09-14T04:12:53.378955+0000","last_fullsized":"2024-09-14T04:12:53.378955+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":312,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.367553+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:12:53.367553+0000","last_peered":"2024-09-14T04:12:53.367553+0000","last_clean":"2024-09-14T04:12:53.367553+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:12:53.367553+0000","last_undegraded":"2024-09-14T04:12:53.367553+0000","last_fullsized":"2024-09-14T04:12:53.367553+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"236'13965","reported_seq":15167,"reported_epoch":236,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:12:56.380232+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:12:56.380232+0000","last_peered":"2024-09-14T04:12:56.380232+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:12:56.380232+0000","last_undegraded":"2024-09-14T04:12:56.380232+0000","last_fullsized":"2024-09-14T04:12:56.380232+0000","mapping_epoch":210,"log_start":"208'7400","ondisk_log_start":"208'7400","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21626904,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":331,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13965,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":331}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":483,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:51.415440+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:12:51.415440+0000","last_peered":"2024-09-14T04:12:51.415440+0000","last_clean":"2024-09-14T04:12:51.415440+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:12:51.415440+0000","last_undegraded":"2024-09-14T04:12:51.415440+0000","last_fullsized":"2024-09-14T04:12:51.415440+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":294,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.378851+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:12:53.378851+0000","last_peered":"2024-09-14T04:12:53.378851+0000","last_clean":"2024-09-14T04:12:53.378851+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:12:53.378851+0000","last_undegraded":"2024-09-14T04:12:53.378851+0000","last_fullsized":"2024-09-14T04:12:53.378851+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":252,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.336142+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:12:46.336142+0000","last_peered":"2024-09-14T04:12:46.336142+0000","last_clean":"2024-09-14T04:12:46.336142+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:12:46.336142+0000","last_undegraded":"2024-09-14T04:12:46.336142+0000","last_fullsized":"2024-09-14T04:12:46.336142+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349267+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:12:53.349267+0000","last_peered":"2024-09-14T04:12:53.349267+0000","last_clean":"2024-09-14T04:12:53.349267+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:12:53.349267+0000","last_undegraded":"2024-09-14T04:12:53.349267+0000","last_fullsized":"2024-09-14T04:12:53.349267+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"236'13838","reported_seq":13900,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:55.863189+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:12:55.863189+0000","last_peered":"2024-09-14T04:12:55.863189+0000","last_clean":"2024-09-14T04:12:55.863189+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:12:55.863189+0000","last_undegraded":"2024-09-14T04:12:55.863189+0000","last_fullsized":"2024-09-14T04:12:55.863189+0000","mapping_epoch":184,"log_start":"207'6512","ondisk_log_start":"207'6512","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25690112,"num_objects":392,"num_object_clones":0,"num_object_copies":1176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13838,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":441,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269338+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:12:46.269338+0000","last_peered":"2024-09-14T04:12:46.269338+0000","last_clean":"2024-09-14T04:12:46.269338+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:12:46.269338+0000","last_undegraded":"2024-09-14T04:12:46.269338+0000","last_fullsized":"2024-09-14T04:12:46.269338+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":170,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335929+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:12:46.335929+0000","last_peered":"2024-09-14T04:12:46.335929+0000","last_clean":"2024-09-14T04:12:46.335929+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:12:46.335929+0000","last_undegraded":"2024-09-14T04:12:46.335929+0000","last_fullsized":"2024-09-14T04:12:46.335929+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":294,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.369294+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:12:53.369294+0000","last_peered":"2024-09-14T04:12:53.369294+0000","last_clean":"2024-09-14T04:12:53.369294+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:12:53.369294+0000","last_undegraded":"2024-09-14T04:12:53.369294+0000","last_fullsized":"2024-09-14T04:12:53.369294+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335496+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:12:46.335496+0000","last_peered":"2024-09-14T04:12:46.335496+0000","last_clean":"2024-09-14T04:12:46.335496+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:12:46.335496+0000","last_undegraded":"2024-09-14T04:12:46.335496+0000","last_fullsized":"2024-09-14T04:12:46.335496+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"236'13941","reported_seq":14003,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:56.380225+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:12:56.380225+0000","last_peered":"2024-09-14T04:12:56.380225+0000","last_clean":"2024-09-14T04:12:56.380225+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:12:56.380225+0000","last_undegraded":"2024-09-14T04:12:56.380225+0000","last_fullsized":"2024-09-14T04:12:56.380225+0000","mapping_epoch":184,"log_start":"208'7400","ondisk_log_start":"208'7400","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6541,"log_dups_size":0,"ondisk_log_size":6541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21692416,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13941,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":520,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349989+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:12:53.349989+0000","last_peered":"2024-09-14T04:12:53.349989+0000","last_clean":"2024-09-14T04:12:53.349989+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:12:53.349989+0000","last_undegraded":"2024-09-14T04:12:53.349989+0000","last_fullsized":"2024-09-14T04:12:53.349989+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":69,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347769+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:12:53.347769+0000","last_peered":"2024-09-14T04:12:53.347769+0000","last_clean":"2024-09-14T04:12:53.347769+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:12:53.347769+0000","last_undegraded":"2024-09-14T04:12:53.347769+0000","last_fullsized":"2024-09-14T04:12:53.347769+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":234,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.272272+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:12:46.272272+0000","last_peered":"2024-09-14T04:12:46.272272+0000","last_clean":"2024-09-14T04:12:46.272272+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:12:46.272272+0000","last_undegraded":"2024-09-14T04:12:46.272272+0000","last_fullsized":"2024-09-14T04:12:46.272272+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":190,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268876+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:12:46.268876+0000","last_peered":"2024-09-14T04:12:46.268876+0000","last_clean":"2024-09-14T04:12:46.268876+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:12:46.268876+0000","last_undegraded":"2024-09-14T04:12:46.268876+0000","last_fullsized":"2024-09-14T04:12:46.268876+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":212,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268981+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:12:46.268981+0000","last_peered":"2024-09-14T04:12:46.268981+0000","last_clean":"2024-09-14T04:12:46.268981+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:12:46.268981+0000","last_undegraded":"2024-09-14T04:12:46.268981+0000","last_fullsized":"2024-09-14T04:12:46.268981+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":212,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269444+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:12:46.269444+0000","last_peered":"2024-09-14T04:12:46.269444+0000","last_clean":"2024-09-14T04:12:46.269444+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:12:46.269444+0000","last_undegraded":"2024-09-14T04:12:46.269444+0000","last_fullsized":"2024-09-14T04:12:46.269444+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":294,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.367399+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:12:53.367399+0000","last_peered":"2024-09-14T04:12:53.367399+0000","last_clean":"2024-09-14T04:12:53.367399+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:12:53.367399+0000","last_undegraded":"2024-09-14T04:12:53.367399+0000","last_fullsized":"2024-09-14T04:12:53.367399+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":243,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.335983+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:12:46.335983+0000","last_peered":"2024-09-14T04:12:46.335983+0000","last_clean":"2024-09-14T04:12:46.335983+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:12:46.335983+0000","last_undegraded":"2024-09-14T04:12:46.335983+0000","last_fullsized":"2024-09-14T04:12:46.335983+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":204,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269508+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:12:46.269508+0000","last_peered":"2024-09-14T04:12:46.269508+0000","last_clean":"2024-09-14T04:12:46.269508+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:12:46.269508+0000","last_undegraded":"2024-09-14T04:12:46.269508+0000","last_fullsized":"2024-09-14T04:12:46.269508+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":166,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.339302+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:12:46.339302+0000","last_peered":"2024-09-14T04:12:46.339302+0000","last_clean":"2024-09-14T04:12:46.339302+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:12:46.339302+0000","last_undegraded":"2024-09-14T04:12:46.339302+0000","last_fullsized":"2024-09-14T04:12:46.339302+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349770+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:12:53.349770+0000","last_peered":"2024-09-14T04:12:53.349770+0000","last_clean":"2024-09-14T04:12:53.349770+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:12:53.349770+0000","last_undegraded":"2024-09-14T04:12:53.349770+0000","last_fullsized":"2024-09-14T04:12:53.349770+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347410+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:12:53.347410+0000","last_peered":"2024-09-14T04:12:53.347410+0000","last_clean":"2024-09-14T04:12:53.347410+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:12:53.347410+0000","last_undegraded":"2024-09-14T04:12:53.347410+0000","last_fullsized":"2024-09-14T04:12:53.347410+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.272190+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:12:46.272190+0000","last_peered":"2024-09-14T04:12:46.272190+0000","last_clean":"2024-09-14T04:12:46.272190+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:12:46.272190+0000","last_undegraded":"2024-09-14T04:12:46.272190+0000","last_fullsized":"2024-09-14T04:12:46.272190+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":64,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.351217+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:12:53.351217+0000","last_peered":"2024-09-14T04:12:53.351217+0000","last_clean":"2024-09-14T04:12:53.351217+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:12:53.351217+0000","last_undegraded":"2024-09-14T04:12:53.351217+0000","last_fullsized":"2024-09-14T04:12:53.351217+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":65,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.347670+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:12:53.347670+0000","last_peered":"2024-09-14T04:12:53.347670+0000","last_clean":"2024-09-14T04:12:53.347670+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:12:53.347670+0000","last_undegraded":"2024-09-14T04:12:53.347670+0000","last_fullsized":"2024-09-14T04:12:53.347670+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":321,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.367316+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:12:53.367316+0000","last_peered":"2024-09-14T04:12:53.367316+0000","last_clean":"2024-09-14T04:12:53.367316+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:12:53.367316+0000","last_undegraded":"2024-09-14T04:12:53.367316+0000","last_fullsized":"2024-09-14T04:12:53.367316+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269035+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:12:46.269035+0000","last_peered":"2024-09-14T04:12:46.269035+0000","last_clean":"2024-09-14T04:12:46.269035+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:12:46.269035+0000","last_undegraded":"2024-09-14T04:12:46.269035+0000","last_fullsized":"2024-09-14T04:12:46.269035+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":218,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.348145+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:12:53.348145+0000","last_peered":"2024-09-14T04:12:53.348145+0000","last_clean":"2024-09-14T04:12:53.348145+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:12:53.348145+0000","last_undegraded":"2024-09-14T04:12:53.348145+0000","last_fullsized":"2024-09-14T04:12:53.348145+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":337,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.373087+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:12:53.373087+0000","last_peered":"2024-09-14T04:12:53.373087+0000","last_clean":"2024-09-14T04:12:53.373087+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:12:53.373087+0000","last_undegraded":"2024-09-14T04:12:53.373087+0000","last_fullsized":"2024-09-14T04:12:53.373087+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":64,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349150+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:12:53.349150+0000","last_peered":"2024-09-14T04:12:53.349150+0000","last_clean":"2024-09-14T04:12:53.349150+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:12:53.349150+0000","last_undegraded":"2024-09-14T04:12:53.349150+0000","last_fullsized":"2024-09-14T04:12:53.349150+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":26,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268813+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:12:46.268813+0000","last_peered":"2024-09-14T04:12:46.268813+0000","last_clean":"2024-09-14T04:12:46.268813+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:12:46.268813+0000","last_undegraded":"2024-09-14T04:12:46.268813+0000","last_fullsized":"2024-09-14T04:12:46.268813+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":243,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268899+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:12:46.268899+0000","last_peered":"2024-09-14T04:12:46.268899+0000","last_clean":"2024-09-14T04:12:46.268899+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:12:46.268899+0000","last_undegraded":"2024-09-14T04:12:46.268899+0000","last_fullsized":"2024-09-14T04:12:46.268899+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":26,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268659+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:12:46.268659+0000","last_peered":"2024-09-14T04:12:46.268659+0000","last_clean":"2024-09-14T04:12:46.268659+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:12:46.268659+0000","last_undegraded":"2024-09-14T04:12:46.268659+0000","last_fullsized":"2024-09-14T04:12:46.268659+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":230,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.269585+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:12:46.269585+0000","last_peered":"2024-09-14T04:12:46.269585+0000","last_clean":"2024-09-14T04:12:46.269585+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:12:46.269585+0000","last_undegraded":"2024-09-14T04:12:46.269585+0000","last_fullsized":"2024-09-14T04:12:46.269585+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":298,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.379067+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:12:53.379067+0000","last_peered":"2024-09-14T04:12:53.379067+0000","last_clean":"2024-09-14T04:12:53.379067+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:12:53.379067+0000","last_undegraded":"2024-09-14T04:12:53.379067+0000","last_fullsized":"2024-09-14T04:12:53.379067+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":112,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.272020+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:12:46.272020+0000","last_peered":"2024-09-14T04:12:46.272020+0000","last_clean":"2024-09-14T04:12:46.272020+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:12:46.272020+0000","last_undegraded":"2024-09-14T04:12:46.272020+0000","last_fullsized":"2024-09-14T04:12:46.272020+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+clean","last_fresh":"2024-09-14T04:12:46.268815+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:12:46.268815+0000","last_peered":"2024-09-14T04:12:46.268815+0000","last_clean":"2024-09-14T04:12:46.268815+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:12:46.268815+0000","last_undegraded":"2024-09-14T04:12:46.268815+0000","last_fullsized":"2024-09-14T04:12:46.268815+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":71,"reported_epoch":236,"state":"active+clean","last_fresh":"2024-09-14T04:12:53.349888+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:12:53.349888+0000","last_peered":"2024-09-14T04:12:53.349888+0000","last_clean":"2024-09-14T04:12:53.349888+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:12:53.349888+0000","last_undegraded":"2024-09-14T04:12:53.349888+0000","last_fullsized":"2024-09-14T04:12:53.349888+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":363855896,"num_objects":5553,"num_object_clones":0,"num_object_copies":16659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4800,"num_objects_misplaced":8246,"num_objects_unfound":0,"num_objects_dirty":5553,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224723,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14949,"num_bytes_recovered":801046528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1994797056,"data_stored":1994784840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119980,"ondisk_log_size":119980,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201668,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997104,"kb_used_data":226024,"kb_used_omap":1,"kb_used_meta":771070,"kb_avail":92748048,"statfs":{"total":95995035648,"available":94974001152,"internally_reserved":0,"allocated":231448576,"data_stored":229944157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":789576139},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463306,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828784,"kb_used_data":161896,"kb_used_omap":1,"kb_used_meta":666878,"kb_avail":92916368,"statfs":{"total":95995035648,"available":95146360832,"internally_reserved":0,"allocated":165781504,"data_stored":164277588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":682883527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724942,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122788,"kb_used_data":121180,"kb_used_omap":1,"kb_used_meta":1001598,"kb_avail":92622364,"statfs":{"total":95995035648,"available":94845300736,"internally_reserved":0,"allocated":124088320,"data_stored":122592185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1025636806},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986578,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482416,"kb_used_data":657192,"kb_used_omap":1,"kb_used_meta":825214,"kb_avail":92262736,"statfs":{"total":95995035648,"available":94477041664,"internally_reserved":0,"allocated":672964608,"data_stored":671459904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":845019590},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748607,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":562724,"kb_used_data":146012,"kb_used_omap":4,"kb_used_meta":416571,"kb_avail":93182428,"statfs":{"total":95995035648,"available":95418806272,"internally_reserved":0,"allocated":149516288,"data_stored":148019889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":426569260},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912134,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":756068,"kb_used_data":496924,"kb_used_omap":6,"kb_used_meta":259065,"kb_avail":92989084,"statfs":{"total":95995035648,"available":95220822016,"internally_reserved":0,"allocated":508850176,"data_stored":507354279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":265283129},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468150,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":620312,"kb_used_data":116304,"kb_used_omap":5,"kb_used_meta":503930,"kb_avail":93124840,"statfs":{"total":95995035648,"available":95359836160,"internally_reserved":0,"allocated":119095296,"data_stored":117609126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":516024671},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342266,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":322468,"kb_used_data":116188,"kb_used_omap":10,"kb_used_meta":206197,"kb_avail":93422684,"statfs":{"total":95995035648,"available":95664828416,"internally_reserved":0,"allocated":118976512,"data_stored":117480093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":211145938},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":109510656,"data_stored":109510656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":112459776,"data_stored":112459776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":489881600,"data_stored":489881600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":128716800,"data_stored":128712728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":659755008,"data_stored":659750936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":111869952,"data_stored":111869952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":161681408,"data_stored":161677336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":220921856,"data_stored":220921856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:13:01.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:01.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:02.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:02.078 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:02.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:02.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:02.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:02.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:02.230 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:13:02.230 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:13:02.267 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:13:02.268 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:13:02.305 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:13:02.305 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:13:02.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:02.345 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:13:02.345 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:13:02.352 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:02.379 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:13:02.379 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:13:02.414 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:13:02.415 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:13:02.415 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:02 smithi001 ceph-mon[107399]: pgmap v820: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 271 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 4800/13065 objects degraded (36.739%); 8246/13065 objects misplaced (63.115%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:02.415 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4115423788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:02.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:02 smithi001 ceph-mon[112165]: pgmap v820: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 271 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 4800/13065 objects degraded (36.739%); 8246/13065 objects misplaced (63.115%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:02.415 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4115423788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:02.448 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:13:02.448 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:13:02.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:02.491 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:13:02.491 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:13:02.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:02.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:02.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:02.651 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:02.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:02.753 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:02.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:02 smithi012 ceph-mon[101180]: pgmap v820: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 271 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 4800/13065 objects degraded (36.739%); 8246/13065 objects misplaced (63.115%); 3.1 MiB/s, 80 objects/s recovering 2024-09-14T04:13:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4115423788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:02.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:02.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:02.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:03.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:03.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:03.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:03.200 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:03.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:03.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:03.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:03.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:03.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:03.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:03.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 115137 objects 2024-09-14T04:13:03.627 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :156.978 2024-09-14T04:13:03.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:03.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:03.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:03.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:03.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:03.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:04.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:04.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:04.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:04.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:04.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:04.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:04.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:04.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:04.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:04.502 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:04.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:04.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:04.609 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:04 smithi001 ceph-mon[107399]: pgmap v821: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 119 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4651/5778 objects degraded (80.495%); 7515/5778 objects misplaced (130.062%); 3.2 MiB/s, 83 objects/s recovering 2024-09-14T04:13:04.609 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4651/5778 objects degraded (80.495%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:04.610 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:04 smithi001 ceph-mon[112165]: pgmap v821: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 119 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4651/5778 objects degraded (80.495%); 7515/5778 objects misplaced (130.062%); 3.2 MiB/s, 83 objects/s recovering 2024-09-14T04:13:04.610 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4651/5778 objects degraded (80.495%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:04.628 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:04.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:04.752 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:04 smithi012 ceph-mon[101180]: pgmap v821: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 119 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4651/5778 objects degraded (80.495%); 7515/5778 objects misplaced (130.062%); 3.2 MiB/s, 83 objects/s recovering 2024-09-14T04:13:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4651/5778 objects degraded (80.495%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:04.854 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:04.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:04.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:04.924 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:13:04.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:05.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:05.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:05.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:05.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:05.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:05.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:05.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:05.358 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:13:05.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:05.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:05.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:05.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:05.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:13:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:13:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:05.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:05.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:05.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:13:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:05.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:05.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:05.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:05.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:06.008 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:06.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:06.133 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:06.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:06.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:06.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:06.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:06.383 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:06.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:06.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:06.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:06.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[107399]: pgmap v822: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 80 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 4651/3891 objects degraded (119.532%); 7515/3891 objects misplaced (193.138%); 2.7 MiB/s, 70 objects/s recovering 2024-09-14T04:13:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:13:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:13:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[107399]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T04:13:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[112165]: pgmap v822: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 80 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 4651/3891 objects degraded (119.532%); 7515/3891 objects misplaced (193.138%); 2.7 MiB/s, 70 objects/s recovering 2024-09-14T04:13:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:13:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:13:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-mon[112165]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T04:13:06.717 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-14T04:13:06.717 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-14T04:13:06.735 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:06.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:06 smithi012 ceph-mon[101180]: pgmap v822: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 80 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 4651/3891 objects degraded (119.532%); 7515/3891 objects misplaced (193.138%); 2.7 MiB/s, 70 objects/s recovering 2024-09-14T04:13:06.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:06 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:13:06.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1864507744' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:13:06.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:06 smithi012 ceph-mon[101180]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T04:13:06.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:06.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:06.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:06.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:06] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-14T04:13:07.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:07.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:07.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:07.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:07.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:07.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:07.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:07.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:07.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:07.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:07.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:07.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:07.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:07.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:07.804 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:07.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:07.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:08.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:08.061 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:08.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:08.186 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:08.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:08.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:08.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:08.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:08.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:08.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:08.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:08.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:08.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:08 smithi001 ceph-mon[107399]: pgmap v824: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 8.9 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4651/480 objects degraded (968.958%); 7270/480 objects misplaced (1514.583%); 1.7 MiB/s, 124 objects/s recovering 2024-09-14T04:13:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:08 smithi001 ceph-mon[112165]: pgmap v824: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 8.9 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4651/480 objects degraded (968.958%); 7270/480 objects misplaced (1514.583%); 1.7 MiB/s, 124 objects/s recovering 2024-09-14T04:13:08.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:08 smithi012 ceph-mon[101180]: pgmap v824: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 8.9 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4651/480 objects degraded (968.958%); 7270/480 objects misplaced (1514.583%); 1.7 MiB/s, 124 objects/s recovering 2024-09-14T04:13:08.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:08.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:08.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:08.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:08.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:09.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:09.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:09.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:09.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:09.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:09.317 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:09.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:09.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:09.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:09.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:09.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:09.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:09.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4651/480 objects degraded (968.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4651/480 objects degraded (968.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:09.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4651/480 objects degraded (968.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:09.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:09.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:09.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:10.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:10.067 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:10.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:10.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:10.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:10.315 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:10.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:10.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:10.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:10.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:10.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:10.566 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:10.622 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:10 smithi001 ceph-mon[107399]: pgmap v825: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:10.622 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:10 smithi001 ceph-mon[112165]: pgmap v825: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:10.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:10.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:10.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:10.797 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:10.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:10 smithi012 ceph-mon[101180]: pgmap v825: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:10.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:10.946 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:10.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:11.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:11.072 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:11.174 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:11.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:11.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:11.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:11.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:11.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:11.447 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:11.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:11.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:11.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:11.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:11.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:11.826 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:11.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:11.949 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:11.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:12.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:12.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:12.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:12.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:12.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:12.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:12.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:12.454 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:12.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:12.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:12.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:12 smithi001 ceph-mon[107399]: pgmap v826: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:12 smithi001 ceph-mon[112165]: pgmap v826: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:12.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:12.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:12 smithi012 ceph-mon[101180]: pgmap v826: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 4114/78 objects degraded (5274.359%); 7270/78 objects misplaced (9320.513%); 1.7 MiB/s, 161 objects/s recovering 2024-09-14T04:13:12.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:12.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:12.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:13.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:13.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:13.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:13.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:13.310 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:13.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:13.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:13.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:13.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:13.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:13.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:13.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:13.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:13.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:13 smithi001 ceph-mon[107399]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T04:13:13.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:13 smithi001 ceph-mon[112165]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T04:13:13.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:13.835 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:13.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:13 smithi012 ceph-mon[101180]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T04:13:13.938 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:13.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:13.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:14.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:14.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:14.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:14.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:14.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:14.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:14.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:14.461 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:14.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:14.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:14.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:14.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:14.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[112165]: pgmap v828: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 2574/78 objects degraded (3300.000%); 7270/78 objects misplaced (9320.513%); 120 KiB/s, 508 objects/s recovering 2024-09-14T04:13:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[112165]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T04:13:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2574/78 objects degraded (3300.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[107399]: pgmap v828: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 2574/78 objects degraded (3300.000%); 7270/78 objects misplaced (9320.513%); 120 KiB/s, 508 objects/s recovering 2024-09-14T04:13:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[107399]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T04:13:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2574/78 objects degraded (3300.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:14.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:14.840 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:14 smithi012 ceph-mon[101180]: pgmap v828: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 2574/78 objects degraded (3300.000%); 7270/78 objects misplaced (9320.513%); 120 KiB/s, 508 objects/s recovering 2024-09-14T04:13:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:14 smithi012 ceph-mon[101180]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T04:13:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2574/78 objects degraded (3300.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:13:14.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:14.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:14.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:14.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:15.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:15.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:15.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:15.217 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:15.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:15.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:15.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:15.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:15.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:15.592 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:15.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:15.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:15.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:15.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:15.842 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:15.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:15.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:15.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:15.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:16.068 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:16.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:16.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:16.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:16.343 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:16.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:16.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:16.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:16.590 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:16.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:16.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:16 smithi001 ceph-mon[107399]: pgmap v830: 99 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 2574/78 objects degraded (3300.000%); 6863/78 objects misplaced (8798.718%); 0 B/s, 455 objects/s recovering 2024-09-14T04:13:16.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:16 smithi001 ceph-mon[112165]: pgmap v830: 99 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 2574/78 objects degraded (3300.000%); 6863/78 objects misplaced (8798.718%); 0 B/s, 455 objects/s recovering 2024-09-14T04:13:16.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:16.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:16.839 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:16.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:16 smithi012 ceph-mon[101180]: pgmap v830: 99 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 94 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 2574/78 objects degraded (3300.000%); 6863/78 objects misplaced (8798.718%); 0 B/s, 455 objects/s recovering 2024-09-14T04:13:16.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:16.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:16.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:16.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:16] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-14T04:13:17.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:17.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:17.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:17.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:17.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:17.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:17.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:17.468 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:17.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:17.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:17.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:17.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:17.824 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:17.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:17.955 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:17.978 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:17.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:18.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:18.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:18.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:18.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:18.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:18.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:18.353 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:18.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:18.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:18.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:18.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:18.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:18.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:18 smithi001 ceph-mon[107399]: pgmap v831: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2474/78 objects degraded (3171.795%); 6863/78 objects misplaced (8798.718%); 0 B/s, 401 objects/s recovering 2024-09-14T04:13:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:18 smithi001 ceph-mon[112165]: pgmap v831: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2474/78 objects degraded (3171.795%); 6863/78 objects misplaced (8798.718%); 0 B/s, 401 objects/s recovering 2024-09-14T04:13:18.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:18.792 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:18 smithi012 ceph-mon[101180]: pgmap v831: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 2474/78 objects degraded (3171.795%); 6863/78 objects misplaced (8798.718%); 0 B/s, 401 objects/s recovering 2024-09-14T04:13:18.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:18.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:18.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:18.980 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:18.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:19.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:19.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:19.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:19.228 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:19.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:19.352 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:19.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:19.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:19.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:19.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2474/78 objects degraded (3171.795%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2474/78 objects degraded (3171.795%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:19.725 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:19.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:13:19.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2474/78 objects degraded (3171.795%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:19.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:19.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:19.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:19.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:20.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:20.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:20.161 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:20.161 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:20.172 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:13:20.173 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:13:20.173 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:13:20.173 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:13:20.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:20.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:20.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:20.351 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:20.453 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:20.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:20.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:20.604 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:20.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:20 smithi001 ceph-mon[107399]: pgmap v832: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6863/78 objects misplaced (8798.718%); 0 B/s, 517 objects/s recovering 2024-09-14T04:13:20.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:20.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:20 smithi001 ceph-mon[112165]: pgmap v832: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6863/78 objects misplaced (8798.718%); 0 B/s, 517 objects/s recovering 2024-09-14T04:13:20.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:20.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:20.730 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:20.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:20 smithi012 ceph-mon[101180]: pgmap v832: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6863/78 objects misplaced (8798.718%); 0 B/s, 517 objects/s recovering 2024-09-14T04:13:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:20.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:20.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:20.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:20.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:21.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:13:21.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:21.107 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:21.208 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:21.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:13:21.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:21.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:21.367 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:21.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:21.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:21.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:21.614 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:21.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:21.740 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:21.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:21.864 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:21.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:21.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:21.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:22.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:22.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:22.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:22.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:22.240 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:22.342 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:22.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:22.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:22.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:22.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:22.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:22.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:22.619 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:22.718 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:22 smithi001 ceph-mon[107399]: pgmap v833: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6676/78 objects misplaced (8558.974%); 0 B/s, 260 objects/s recovering 2024-09-14T04:13:22.718 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:22 smithi001 ceph-mon[112165]: pgmap v833: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6676/78 objects misplaced (8558.974%); 0 B/s, 260 objects/s recovering 2024-09-14T04:13:22.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:22.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:22.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:22.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:22 smithi012 ceph-mon[101180]: pgmap v833: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 1544/78 objects degraded (1979.487%); 6676/78 objects misplaced (8558.974%); 0 B/s, 260 objects/s recovering 2024-09-14T04:13:22.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:22.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:22.994 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:22.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:23.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:23.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:23.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:23.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:23.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:23.346 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:23.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:23.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:23.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:23.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:23.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:23.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:23.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:23.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:23.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:23.868 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:23.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:23.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:23.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:24.094 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:24.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:24.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:24.243 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:24.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:24.360 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:24.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:24.472 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:24.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:24.517 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:24.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:24.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[107399]: pgmap v834: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6676/78 objects misplaced (8558.974%); 0 B/s, 289 objects/s recovering 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[107399]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[112165]: pgmap v834: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6676/78 objects misplaced (8558.974%); 0 B/s, 289 objects/s recovering 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:24 smithi001 ceph-mon[112165]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T04:13:24.723 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:24.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:24 smithi012 ceph-mon[101180]: pgmap v834: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6676/78 objects misplaced (8558.974%); 0 B/s, 289 objects/s recovering 2024-09-14T04:13:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:13:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:24 smithi012 ceph-mon[101180]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T04:13:24.848 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:24.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:24.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:24.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:24.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:25.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:25.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:25.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:25.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:25.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:25.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:25.476 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:25.499 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:25.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:25.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:25.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:25.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:25.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:25.872 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:25.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:25.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:25.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:25.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:26.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:26.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:26.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:26.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:26.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:26 smithi012 ceph-mon[101180]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T04:13:26.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:26 smithi012 ceph-mon[101180]: pgmap v837: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6306/78 objects misplaced (8084.615%); 0 B/s, 365 objects/s recovering 2024-09-14T04:13:26.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:26.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:26 smithi001 ceph-mon[107399]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T04:13:26.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:26 smithi001 ceph-mon[107399]: pgmap v837: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6306/78 objects misplaced (8084.615%); 0 B/s, 365 objects/s recovering 2024-09-14T04:13:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:26 smithi001 ceph-mon[112165]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T04:13:26.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:26 smithi001 ceph-mon[112165]: pgmap v837: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 269/78 objects degraded (344.872%); 6306/78 objects misplaced (8084.615%); 0 B/s, 365 objects/s recovering 2024-09-14T04:13:26.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:26.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:26.652 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:26.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:26.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:26.804 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:26.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:26.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:26.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:26] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-09-14T04:13:27.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:27.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:27.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:27.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:27.188 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:27.290 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:27.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:27.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:27.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:27.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:27.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:27 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:13:27.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:27.693 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:27.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:27 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:13:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:27 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 269/78 objects degraded (344.872%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:13:27.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.822 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:27.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:27.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:27.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:27.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:28.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:28.080 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:28.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:28.206 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:28.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:28.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:28.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:28.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:28.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:28.587 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:28 smithi012 ceph-mon[101180]: pgmap v838: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 283 objects/s recovering 2024-09-14T04:13:28.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:28.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:28.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:28 smithi001 ceph-mon[107399]: pgmap v838: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 283 objects/s recovering 2024-09-14T04:13:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:28 smithi001 ceph-mon[112165]: pgmap v838: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 283 objects/s recovering 2024-09-14T04:13:28.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:28.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:28.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:28.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:28.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:28.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:29.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:29.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:29.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:29.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:29.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:29.345 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:29.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:29.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:29.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:29.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:29.603 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:29.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:29.728 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:29.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:29.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:29.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:29.985 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:29.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:30.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:30.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:30.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:30.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:30.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:30.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:30.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:30.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:30.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:30 smithi012 ceph-mon[101180]: pgmap v839: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 239 objects/s recovering 2024-09-14T04:13:30.600 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:30 smithi001 ceph-mon[107399]: pgmap v839: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 239 objects/s recovering 2024-09-14T04:13:30.600 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:30 smithi001 ceph-mon[112165]: pgmap v839: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6306/78 objects misplaced (8084.615%); 0 B/s, 239 objects/s recovering 2024-09-14T04:13:30.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:30.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:30.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:30.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:30.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:30.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:30.993 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:30.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:30.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:31.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:31.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:31.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:31.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:31.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:31.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:31.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:31.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:31.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:31.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:31.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:31.756 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:31.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:31.885 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:31.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:31.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:32.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:32.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:32.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:32.240 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:32.263 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:32.365 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:32.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:32.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:32.521 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:32.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:32 smithi012 ceph-mon[101180]: pgmap v840: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 133 objects/s recovering 2024-09-14T04:13:32.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:32.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:32 smithi001 ceph-mon[107399]: pgmap v840: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 133 objects/s recovering 2024-09-14T04:13:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:32 smithi001 ceph-mon[112165]: pgmap v840: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 133 objects/s recovering 2024-09-14T04:13:32.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:32.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:32.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:32.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:32.905 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:32.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:33.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:33.029 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:33.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:33.154 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:33.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:33.283 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:33.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:33.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:33.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:33.540 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:33.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:33.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:33.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:33.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:33.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:33.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:33.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:33.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:34.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:34.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:34.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:34.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:34.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:34.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:34.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:34.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:34.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:34.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:34.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:34.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:34 smithi012 ceph-mon[101180]: pgmap v841: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 115 objects/s recovering 2024-09-14T04:13:34.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:34.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:34.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:34 smithi001 ceph-mon[107399]: pgmap v841: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 115 objects/s recovering 2024-09-14T04:13:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:34 smithi001 ceph-mon[112165]: pgmap v841: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5880/78 objects misplaced (7538.462%); 0 B/s, 115 objects/s recovering 2024-09-14T04:13:34.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:34.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:34.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:34.929 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:34.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:35.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:35.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:35.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:35.178 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:35.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:35.307 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:35.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:35.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:35.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:35.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:35.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:35.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:35.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:35.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:35.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:35.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:35.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:35.945 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:35.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:36.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:36.074 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:36.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:36.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:36.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:36.179 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[107399]: pgmap v842: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 110 objects/s recovering 2024-09-14T04:13:36.180 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:36.180 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:36.180 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:35 smithi001 ceph-mon[112165]: pgmap v842: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 110 objects/s recovering 2024-09-14T04:13:36.199 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:36.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:36.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:35 smithi012 ceph-mon[101180]: pgmap v842: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 110 objects/s recovering 2024-09-14T04:13:36.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:36.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:36.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:36.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:36.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:36.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:36.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:36.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:36.835 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:36.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:36.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:36.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:36.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:36] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-14T04:13:37.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:37.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:37.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:37.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:37.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:37.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:37.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:37.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:37.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:37.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:37.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:37.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:37.724 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:37.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:37.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:37.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:37.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:37.983 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:37.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:38.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:38.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:38.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:38.238 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:38.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:38.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:38.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:38.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:38.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:38 smithi012 ceph-mon[101180]: pgmap v843: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 94 objects/s recovering 2024-09-14T04:13:38.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:38.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:38 smithi001 ceph-mon[107399]: pgmap v843: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 94 objects/s recovering 2024-09-14T04:13:38.617 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:38 smithi001 ceph-mon[112165]: pgmap v843: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 94 objects/s recovering 2024-09-14T04:13:38.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:38.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:38.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:38.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:38.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:38.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:38.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:39.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:39.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:39.226 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:39.249 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:39.273 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:13:39.274 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:13:39.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:39.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:39.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:39.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:39.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:39.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:39.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:39.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:39.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:39.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:39.884 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:39.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:39.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:40.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:40.108 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":843,"stamp":"2024-09-14T04:13:37.208823+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5446,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":230340,"num_write_kb":7368802,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23865,"num_bytes_recovered":819987922,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121731,"ondisk_log_size":121731,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5050564,"kb_used_data":358372,"kb_used_omap":33,"kb_used_meta":4692062,"kb_avail":744910652,"statfs":{"total":767960285184,"available":762788507648,"internally_reserved":0,"allocated":366972928,"data_stored":354695631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4804671924},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-269,"num_objects_misplaced":-860,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1129,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002617"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.030221+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:13:25.030221+0000","last_peered":"2024-09-14T04:13:25.030221+0000","last_clean":"2024-09-14T04:13:25.030221+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:13:25.030221+0000","last_undegraded":"2024-09-14T04:13:25.030221+0000","last_fullsized":"2024-09-14T04:13:25.030221+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":225,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028813+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:13:25.028813+0000","last_peered":"2024-09-14T04:13:25.028813+0000","last_clean":"2024-09-14T04:13:25.028813+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:13:25.028813+0000","last_undegraded":"2024-09-14T04:13:25.028813+0000","last_fullsized":"2024-09-14T04:13:25.028813+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026290+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:13:24.026290+0000","last_peered":"2024-09-14T04:13:24.026290+0000","last_clean":"2024-09-14T04:13:24.026290+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:13:24.026290+0000","last_undegraded":"2024-09-14T04:13:24.026290+0000","last_fullsized":"2024-09-14T04:13:24.026290+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029018+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:13:25.029018+0000","last_peered":"2024-09-14T04:13:25.029018+0000","last_clean":"2024-09-14T04:13:25.029018+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:13:25.029018+0000","last_undegraded":"2024-09-14T04:13:25.029018+0000","last_fullsized":"2024-09-14T04:13:25.029018+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028839+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:13:25.028839+0000","last_peered":"2024-09-14T04:13:25.028839+0000","last_clean":"2024-09-14T04:13:25.028839+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:13:25.028839+0000","last_undegraded":"2024-09-14T04:13:25.028839+0000","last_fullsized":"2024-09-14T04:13:25.028839+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":195,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028812+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:13:24.028812+0000","last_peered":"2024-09-14T04:13:24.028812+0000","last_clean":"2024-09-14T04:13:24.028812+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:13:24.028812+0000","last_undegraded":"2024-09-14T04:13:24.028812+0000","last_fullsized":"2024-09-14T04:13:24.028812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028043+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:13:25.028043+0000","last_peered":"2024-09-14T04:13:25.028043+0000","last_clean":"2024-09-14T04:13:25.028043+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:13:25.028043+0000","last_undegraded":"2024-09-14T04:13:25.028043+0000","last_fullsized":"2024-09-14T04:13:25.028043+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":292,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156423+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:13:24.156423+0000","last_peered":"2024-09-14T04:13:24.156423+0000","last_clean":"2024-09-14T04:13:24.156423+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:13:24.156423+0000","last_undegraded":"2024-09-14T04:13:24.156423+0000","last_fullsized":"2024-09-14T04:13:24.156423+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"236'14814","reported_seq":14882,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026582+0000","last_change":"2024-09-14T04:08:44.168594+0000","last_active":"2024-09-14T04:13:24.026582+0000","last_peered":"2024-09-14T04:13:24.026582+0000","last_clean":"2024-09-14T04:13:24.026582+0000","last_became_active":"2024-09-14T04:08:44.168276+0000","last_became_peered":"2024-09-14T04:08:44.168276+0000","last_unstale":"2024-09-14T04:13:24.026582+0000","last_undegraded":"2024-09-14T04:13:24.026582+0000","last_fullsized":"2024-09-14T04:13:24.026582+0000","mapping_epoch":184,"log_start":"199'5100","ondisk_log_start":"199'5100","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":9714,"log_dups_size":0,"ondisk_log_size":9714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:01.434322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14814,"num_write_kb":474048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":252,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029744+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:13:24.029744+0000","last_peered":"2024-09-14T04:13:24.029744+0000","last_clean":"2024-09-14T04:13:24.029744+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:13:24.029744+0000","last_undegraded":"2024-09-14T04:13:24.029744+0000","last_fullsized":"2024-09-14T04:13:24.029744+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026065+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:13:24.026065+0000","last_peered":"2024-09-14T04:13:24.026065+0000","last_clean":"2024-09-14T04:13:24.026065+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:13:24.026065+0000","last_undegraded":"2024-09-14T04:13:24.026065+0000","last_fullsized":"2024-09-14T04:13:24.026065+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"236'14572","reported_seq":26721,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156728+0000","last_change":"2024-09-14T04:12:46.311123+0000","last_active":"2024-09-14T04:13:24.156728+0000","last_peered":"2024-09-14T04:13:24.156728+0000","last_clean":"2024-09-14T04:13:24.156728+0000","last_became_active":"2024-09-14T04:12:46.274363+0000","last_became_peered":"2024-09-14T04:12:46.274363+0000","last_unstale":"2024-09-14T04:13:24.156728+0000","last_undegraded":"2024-09-14T04:13:24.156728+0000","last_fullsized":"2024-09-14T04:13:24.156728+0000","mapping_epoch":234,"log_start":"214'8000","ondisk_log_start":"214'8000","created":184,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:52:16.466037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14572,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6022,"num_bytes_recovered":272302080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":251,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029783+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:13:24.029783+0000","last_peered":"2024-09-14T04:13:24.029783+0000","last_clean":"2024-09-14T04:13:24.029783+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:13:24.029783+0000","last_undegraded":"2024-09-14T04:13:24.029783+0000","last_fullsized":"2024-09-14T04:13:24.029783+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028888+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:13:24.028888+0000","last_peered":"2024-09-14T04:13:24.028888+0000","last_clean":"2024-09-14T04:13:24.028888+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:13:24.028888+0000","last_undegraded":"2024-09-14T04:13:24.028888+0000","last_fullsized":"2024-09-14T04:13:24.028888+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"236'14542","reported_seq":19506,"reported_epoch":241,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:13:34.980865+0000","last_change":"2024-09-14T04:13:11.586419+0000","last_active":"2024-09-14T04:13:34.980865+0000","last_peered":"2024-09-14T04:13:34.980865+0000","last_clean":"2024-09-14T04:10:34.328575+0000","last_became_active":"2024-09-14T04:10:36.607758+0000","last_became_peered":"2024-09-14T04:10:36.607758+0000","last_unstale":"2024-09-14T04:13:34.980865+0000","last_undegraded":"2024-09-14T04:13:34.980865+0000","last_fullsized":"2024-09-14T04:13:34.980865+0000","mapping_epoch":210,"log_start":"207'6600","ondisk_log_start":"207'6600","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7942,"log_dups_size":0,"ondisk_log_size":7942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5446,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14542,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2418,"num_bytes_recovered":38862848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":-5446}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":222,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029447+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:13:24.029447+0000","last_peered":"2024-09-14T04:13:24.029447+0000","last_clean":"2024-09-14T04:13:24.029447+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:13:24.029447+0000","last_undegraded":"2024-09-14T04:13:24.029447+0000","last_fullsized":"2024-09-14T04:13:24.029447+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":208,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156413+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:13:24.156413+0000","last_peered":"2024-09-14T04:13:24.156413+0000","last_clean":"2024-09-14T04:13:24.156413+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:13:24.156413+0000","last_undegraded":"2024-09-14T04:13:24.156413+0000","last_fullsized":"2024-09-14T04:13:24.156413+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"236'14516","reported_seq":14643,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.030230+0000","last_change":"2024-09-14T04:08:44.228058+0000","last_active":"2024-09-14T04:13:25.030230+0000","last_peered":"2024-09-14T04:13:25.030230+0000","last_clean":"2024-09-14T04:13:25.030230+0000","last_became_active":"2024-09-14T04:08:44.227688+0000","last_became_peered":"2024-09-14T04:08:44.227688+0000","last_unstale":"2024-09-14T04:13:25.030230+0000","last_undegraded":"2024-09-14T04:13:25.030230+0000","last_fullsized":"2024-09-14T04:13:25.030230+0000","mapping_epoch":184,"log_start":"206'6100","ondisk_log_start":"206'6100","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8416,"log_dups_size":0,"ondisk_log_size":8416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:34.289752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14516,"num_write_kb":464512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":228,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026490+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:13:24.026490+0000","last_peered":"2024-09-14T04:13:24.026490+0000","last_clean":"2024-09-14T04:13:24.026490+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:13:24.026490+0000","last_undegraded":"2024-09-14T04:13:24.026490+0000","last_fullsized":"2024-09-14T04:13:24.026490+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028924+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:13:25.028924+0000","last_peered":"2024-09-14T04:13:25.028924+0000","last_clean":"2024-09-14T04:13:25.028924+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:13:25.028924+0000","last_undegraded":"2024-09-14T04:13:25.028924+0000","last_fullsized":"2024-09-14T04:13:25.028924+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"236'14182","reported_seq":21078,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041047+0000","last_change":"2024-09-14T04:11:10.007594+0000","last_active":"2024-09-14T04:13:25.041047+0000","last_peered":"2024-09-14T04:13:25.041047+0000","last_clean":"2024-09-14T04:13:25.041047+0000","last_became_active":"2024-09-14T04:11:09.992930+0000","last_became_peered":"2024-09-14T04:11:09.992930+0000","last_unstale":"2024-09-14T04:13:25.041047+0000","last_undegraded":"2024-09-14T04:13:25.041047+0000","last_fullsized":"2024-09-14T04:13:25.041047+0000","mapping_epoch":217,"log_start":"207'6200","ondisk_log_start":"207'6200","created":184,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7982,"log_dups_size":0,"ondisk_log_size":7982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:13.056401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14182,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3376,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":593,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028733+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:13:24.028733+0000","last_peered":"2024-09-14T04:13:24.028733+0000","last_clean":"2024-09-14T04:13:24.028733+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:13:24.028733+0000","last_undegraded":"2024-09-14T04:13:24.028733+0000","last_fullsized":"2024-09-14T04:13:24.028733+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":79,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028381+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:13:25.028381+0000","last_peered":"2024-09-14T04:13:25.028381+0000","last_clean":"2024-09-14T04:13:25.028381+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:13:25.028381+0000","last_undegraded":"2024-09-14T04:13:25.028381+0000","last_fullsized":"2024-09-14T04:13:25.028381+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028744+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:13:24.028744+0000","last_peered":"2024-09-14T04:13:24.028744+0000","last_clean":"2024-09-14T04:13:24.028744+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:13:24.028744+0000","last_undegraded":"2024-09-14T04:13:24.028744+0000","last_fullsized":"2024-09-14T04:13:24.028744+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"236'14332","reported_seq":14459,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029920+0000","last_change":"2024-09-14T04:08:44.229606+0000","last_active":"2024-09-14T04:13:25.029920+0000","last_peered":"2024-09-14T04:13:25.029920+0000","last_clean":"2024-09-14T04:13:25.029920+0000","last_became_active":"2024-09-14T04:08:44.229310+0000","last_became_peered":"2024-09-14T04:08:44.229310+0000","last_unstale":"2024-09-14T04:13:25.029920+0000","last_undegraded":"2024-09-14T04:13:25.029920+0000","last_fullsized":"2024-09-14T04:13:25.029920+0000","mapping_epoch":184,"log_start":"206'5900","ondisk_log_start":"206'5900","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8432,"log_dups_size":0,"ondisk_log_size":8432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:40:15.699100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14332,"num_write_kb":458624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":462,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.025995+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:13:24.025995+0000","last_peered":"2024-09-14T04:13:24.025995+0000","last_clean":"2024-09-14T04:13:24.025995+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:13:24.025995+0000","last_undegraded":"2024-09-14T04:13:24.025995+0000","last_fullsized":"2024-09-14T04:13:24.025995+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156554+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:13:24.156554+0000","last_peered":"2024-09-14T04:13:24.156554+0000","last_clean":"2024-09-14T04:13:24.156554+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:13:24.156554+0000","last_undegraded":"2024-09-14T04:13:24.156554+0000","last_fullsized":"2024-09-14T04:13:24.156554+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.212538+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:13:25.212538+0000","last_peered":"2024-09-14T04:13:25.212538+0000","last_clean":"2024-09-14T04:13:25.212538+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:13:25.212538+0000","last_undegraded":"2024-09-14T04:13:25.212538+0000","last_fullsized":"2024-09-14T04:13:25.212538+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"236'14272","reported_seq":24277,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156137+0000","last_change":"2024-09-14T04:13:13.420545+0000","last_active":"2024-09-14T04:13:24.156137+0000","last_peered":"2024-09-14T04:13:24.156137+0000","last_clean":"2024-09-14T04:13:24.156137+0000","last_became_active":"2024-09-14T04:13:13.390668+0000","last_became_peered":"2024-09-14T04:13:13.390668+0000","last_unstale":"2024-09-14T04:13:24.156137+0000","last_undegraded":"2024-09-14T04:13:24.156137+0000","last_fullsized":"2024-09-14T04:13:24.156137+0000","mapping_epoch":238,"log_start":"213'7700","ondisk_log_start":"213'7700","created":184,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:37.198039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14272,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4954,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":32,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.211794+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:13:25.211794+0000","last_peered":"2024-09-14T04:13:25.211794+0000","last_clean":"2024-09-14T04:13:25.211794+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:13:25.211794+0000","last_undegraded":"2024-09-14T04:13:25.211794+0000","last_fullsized":"2024-09-14T04:13:25.211794+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":209,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029433+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:13:24.029433+0000","last_peered":"2024-09-14T04:13:24.029433+0000","last_clean":"2024-09-14T04:13:24.029433+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:13:24.029433+0000","last_undegraded":"2024-09-14T04:13:24.029433+0000","last_fullsized":"2024-09-14T04:13:24.029433+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"236'14426","reported_seq":14494,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029107+0000","last_change":"2024-09-14T04:08:44.169806+0000","last_active":"2024-09-14T04:13:24.029107+0000","last_peered":"2024-09-14T04:13:24.029107+0000","last_clean":"2024-09-14T04:13:24.029107+0000","last_became_active":"2024-09-14T04:08:44.169490+0000","last_became_peered":"2024-09-14T04:08:44.169490+0000","last_unstale":"2024-09-14T04:13:24.029107+0000","last_undegraded":"2024-09-14T04:13:24.029107+0000","last_fullsized":"2024-09-14T04:13:24.029107+0000","mapping_epoch":184,"log_start":"208'7072","ondisk_log_start":"208'7072","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:35.930091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14426,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":446,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026432+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:13:24.026432+0000","last_peered":"2024-09-14T04:13:24.026432+0000","last_clean":"2024-09-14T04:13:24.026432+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:13:24.026432+0000","last_undegraded":"2024-09-14T04:13:24.026432+0000","last_fullsized":"2024-09-14T04:13:24.026432+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":183,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.027091+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:13:25.027091+0000","last_peered":"2024-09-14T04:13:25.027091+0000","last_clean":"2024-09-14T04:13:25.027091+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:13:25.027091+0000","last_undegraded":"2024-09-14T04:13:25.027091+0000","last_fullsized":"2024-09-14T04:13:25.027091+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":318,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041567+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:13:25.041567+0000","last_peered":"2024-09-14T04:13:25.041567+0000","last_clean":"2024-09-14T04:13:25.041567+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:13:25.041567+0000","last_undegraded":"2024-09-14T04:13:25.041567+0000","last_fullsized":"2024-09-14T04:13:25.041567+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"236'14478","reported_seq":14546,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156639+0000","last_change":"2024-09-14T04:08:44.171643+0000","last_active":"2024-09-14T04:13:24.156639+0000","last_peered":"2024-09-14T04:13:24.156639+0000","last_clean":"2024-09-14T04:13:24.156639+0000","last_became_active":"2024-09-14T04:08:44.171325+0000","last_became_peered":"2024-09-14T04:08:44.171325+0000","last_unstale":"2024-09-14T04:13:24.156639+0000","last_undegraded":"2024-09-14T04:13:24.156639+0000","last_fullsized":"2024-09-14T04:13:24.156639+0000","mapping_epoch":184,"log_start":"214'7900","ondisk_log_start":"214'7900","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:56:38.397847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14478,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":611,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028403+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:13:24.028403+0000","last_peered":"2024-09-14T04:13:24.028403+0000","last_clean":"2024-09-14T04:13:24.028403+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:13:24.028403+0000","last_undegraded":"2024-09-14T04:13:24.028403+0000","last_fullsized":"2024-09-14T04:13:24.028403+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":225,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026229+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:13:24.026229+0000","last_peered":"2024-09-14T04:13:24.026229+0000","last_clean":"2024-09-14T04:13:24.026229+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:13:24.026229+0000","last_undegraded":"2024-09-14T04:13:24.026229+0000","last_fullsized":"2024-09-14T04:13:24.026229+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026468+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:13:24.026468+0000","last_peered":"2024-09-14T04:13:24.026468+0000","last_clean":"2024-09-14T04:13:24.026468+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:13:24.026468+0000","last_undegraded":"2024-09-14T04:13:24.026468+0000","last_fullsized":"2024-09-14T04:13:24.026468+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156481+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:13:24.156481+0000","last_peered":"2024-09-14T04:13:24.156481+0000","last_clean":"2024-09-14T04:13:24.156481+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:13:24.156481+0000","last_undegraded":"2024-09-14T04:13:24.156481+0000","last_fullsized":"2024-09-14T04:13:24.156481+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"236'14134","reported_seq":14261,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.027070+0000","last_change":"2024-09-14T04:08:44.169439+0000","last_active":"2024-09-14T04:13:25.027070+0000","last_peered":"2024-09-14T04:13:25.027070+0000","last_clean":"2024-09-14T04:13:25.027070+0000","last_became_active":"2024-09-14T04:08:44.169117+0000","last_became_peered":"2024-09-14T04:08:44.169117+0000","last_unstale":"2024-09-14T04:13:25.027070+0000","last_undegraded":"2024-09-14T04:13:25.027070+0000","last_fullsized":"2024-09-14T04:13:25.027070+0000","mapping_epoch":184,"log_start":"208'7059","ondisk_log_start":"208'7059","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:17.258751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14134,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":502,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041118+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:13:25.041118+0000","last_peered":"2024-09-14T04:13:25.041118+0000","last_clean":"2024-09-14T04:13:25.041118+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:13:25.041118+0000","last_undegraded":"2024-09-14T04:13:25.041118+0000","last_fullsized":"2024-09-14T04:13:25.041118+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":255,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029221+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:13:25.029221+0000","last_peered":"2024-09-14T04:13:25.029221+0000","last_clean":"2024-09-14T04:13:25.029221+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:13:25.029221+0000","last_undegraded":"2024-09-14T04:13:25.029221+0000","last_fullsized":"2024-09-14T04:13:25.029221+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":298,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028442+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:13:25.028442+0000","last_peered":"2024-09-14T04:13:25.028442+0000","last_clean":"2024-09-14T04:13:25.028442+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:13:25.028442+0000","last_undegraded":"2024-09-14T04:13:25.028442+0000","last_fullsized":"2024-09-14T04:13:25.028442+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028937+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:13:25.028937+0000","last_peered":"2024-09-14T04:13:25.028937+0000","last_clean":"2024-09-14T04:13:25.028937+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:13:25.028937+0000","last_undegraded":"2024-09-14T04:13:25.028937+0000","last_fullsized":"2024-09-14T04:13:25.028937+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"236'14434","reported_seq":19053,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.090567+0000","last_change":"2024-09-14T04:13:25.090259+0000","last_active":"2024-09-14T04:13:25.090567+0000","last_peered":"2024-09-14T04:13:25.090567+0000","last_clean":"2024-09-14T04:13:25.090567+0000","last_became_active":"2024-09-14T04:13:25.030531+0000","last_became_peered":"2024-09-14T04:13:25.030531+0000","last_unstale":"2024-09-14T04:13:25.090567+0000","last_undegraded":"2024-09-14T04:13:25.090567+0000","last_fullsized":"2024-09-14T04:13:25.090567+0000","mapping_epoch":240,"log_start":"206'6100","ondisk_log_start":"206'6100","created":184,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":8334,"log_dups_size":0,"ondisk_log_size":8334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:01:25.441536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14434,"num_write_kb":461888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5902,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":473,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028074+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:13:24.028074+0000","last_peered":"2024-09-14T04:13:24.028074+0000","last_clean":"2024-09-14T04:13:24.028074+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:13:24.028074+0000","last_undegraded":"2024-09-14T04:13:24.028074+0000","last_fullsized":"2024-09-14T04:13:24.028074+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":361,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.030119+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:13:25.030119+0000","last_peered":"2024-09-14T04:13:25.030119+0000","last_clean":"2024-09-14T04:13:25.030119+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:13:25.030119+0000","last_undegraded":"2024-09-14T04:13:25.030119+0000","last_fullsized":"2024-09-14T04:13:25.030119+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":304,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041149+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:13:25.041149+0000","last_peered":"2024-09-14T04:13:25.041149+0000","last_clean":"2024-09-14T04:13:25.041149+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:13:25.041149+0000","last_undegraded":"2024-09-14T04:13:25.041149+0000","last_fullsized":"2024-09-14T04:13:25.041149+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":373,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041386+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:13:25.041386+0000","last_peered":"2024-09-14T04:13:25.041386+0000","last_clean":"2024-09-14T04:13:25.041386+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:13:25.041386+0000","last_undegraded":"2024-09-14T04:13:25.041386+0000","last_fullsized":"2024-09-14T04:13:25.041386+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"236'14184","reported_seq":15477,"reported_epoch":240,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:13:24.156692+0000","last_change":"2024-09-14T04:10:36.505814+0000","last_active":"2024-09-14T04:13:24.156692+0000","last_peered":"2024-09-14T04:13:24.156692+0000","last_clean":"2024-09-14T04:10:34.357296+0000","last_became_active":"2024-09-14T04:10:36.495566+0000","last_became_peered":"2024-09-14T04:10:36.495566+0000","last_unstale":"2024-09-14T04:13:24.156692+0000","last_undegraded":"2024-09-14T04:13:24.156692+0000","last_fullsized":"2024-09-14T04:13:24.156692+0000","mapping_epoch":210,"log_start":"212'7600","ondisk_log_start":"212'7600","created":184,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6584,"log_dups_size":0,"ondisk_log_size":6584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:54.060497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14184,"num_write_kb":453888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":598,"num_bytes_recovered":39190528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5,2],"avail_no_missing":["6","0","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":480,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026003+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:13:24.026003+0000","last_peered":"2024-09-14T04:13:24.026003+0000","last_clean":"2024-09-14T04:13:24.026003+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:13:24.026003+0000","last_undegraded":"2024-09-14T04:13:24.026003+0000","last_fullsized":"2024-09-14T04:13:24.026003+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":226,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026548+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:13:24.026548+0000","last_peered":"2024-09-14T04:13:24.026548+0000","last_clean":"2024-09-14T04:13:24.026548+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:13:24.026548+0000","last_undegraded":"2024-09-14T04:13:24.026548+0000","last_fullsized":"2024-09-14T04:13:24.026548+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":244,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028639+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:13:24.028639+0000","last_peered":"2024-09-14T04:13:24.028639+0000","last_clean":"2024-09-14T04:13:24.028639+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:13:24.028639+0000","last_undegraded":"2024-09-14T04:13:24.028639+0000","last_fullsized":"2024-09-14T04:13:24.028639+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026060+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:13:24.026060+0000","last_peered":"2024-09-14T04:13:24.026060+0000","last_clean":"2024-09-14T04:13:24.026060+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:13:24.026060+0000","last_undegraded":"2024-09-14T04:13:24.026060+0000","last_fullsized":"2024-09-14T04:13:24.026060+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"236'14592","reported_seq":14660,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029086+0000","last_change":"2024-09-14T04:08:44.168136+0000","last_active":"2024-09-14T04:13:24.029086+0000","last_peered":"2024-09-14T04:13:24.029086+0000","last_clean":"2024-09-14T04:13:24.029086+0000","last_became_active":"2024-09-14T04:08:44.167784+0000","last_became_peered":"2024-09-14T04:08:44.167784+0000","last_unstale":"2024-09-14T04:13:24.029086+0000","last_undegraded":"2024-09-14T04:13:24.029086+0000","last_fullsized":"2024-09-14T04:13:24.029086+0000","mapping_epoch":184,"log_start":"207'6600","ondisk_log_start":"207'6600","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:38.849125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14592,"num_write_kb":466944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":454,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156766+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:13:24.156766+0000","last_peered":"2024-09-14T04:13:24.156766+0000","last_clean":"2024-09-14T04:13:24.156766+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:13:24.156766+0000","last_undegraded":"2024-09-14T04:13:24.156766+0000","last_fullsized":"2024-09-14T04:13:24.156766+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":249,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026370+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:13:24.026370+0000","last_peered":"2024-09-14T04:13:24.026370+0000","last_clean":"2024-09-14T04:13:24.026370+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:13:24.026370+0000","last_undegraded":"2024-09-14T04:13:24.026370+0000","last_fullsized":"2024-09-14T04:13:24.026370+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":373,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041386+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:13:25.041386+0000","last_peered":"2024-09-14T04:13:25.041386+0000","last_clean":"2024-09-14T04:13:25.041386+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:13:25.041386+0000","last_undegraded":"2024-09-14T04:13:25.041386+0000","last_fullsized":"2024-09-14T04:13:25.041386+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":322,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041266+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:13:25.041266+0000","last_peered":"2024-09-14T04:13:25.041266+0000","last_clean":"2024-09-14T04:13:25.041266+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:13:25.041266+0000","last_undegraded":"2024-09-14T04:13:25.041266+0000","last_fullsized":"2024-09-14T04:13:25.041266+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"236'14296","reported_seq":15504,"reported_epoch":240,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:13:24.156440+0000","last_change":"2024-09-14T04:10:36.508025+0000","last_active":"2024-09-14T04:13:24.156440+0000","last_peered":"2024-09-14T04:13:24.156440+0000","last_clean":"2024-09-14T04:10:34.327992+0000","last_became_active":"2024-09-14T04:10:36.499695+0000","last_became_peered":"2024-09-14T04:10:36.499695+0000","last_unstale":"2024-09-14T04:13:24.156440+0000","last_undegraded":"2024-09-14T04:13:24.156440+0000","last_fullsized":"2024-09-14T04:13:24.156440+0000","mapping_epoch":210,"log_start":"213'7700","ondisk_log_start":"213'7700","created":184,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6596,"log_dups_size":0,"ondisk_log_size":6596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:33.788224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14296,"num_write_kb":457409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,3],"avail_no_missing":["6","0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":488,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156668+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:13:24.156668+0000","last_peered":"2024-09-14T04:13:24.156668+0000","last_clean":"2024-09-14T04:13:24.156668+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:13:24.156668+0000","last_undegraded":"2024-09-14T04:13:24.156668+0000","last_fullsized":"2024-09-14T04:13:24.156668+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":304,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041288+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:13:25.041288+0000","last_peered":"2024-09-14T04:13:25.041288+0000","last_clean":"2024-09-14T04:13:25.041288+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:13:25.041288+0000","last_undegraded":"2024-09-14T04:13:25.041288+0000","last_fullsized":"2024-09-14T04:13:25.041288+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":257,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029519+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:13:24.029519+0000","last_peered":"2024-09-14T04:13:24.029519+0000","last_clean":"2024-09-14T04:13:24.029519+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:13:24.029519+0000","last_undegraded":"2024-09-14T04:13:24.029519+0000","last_fullsized":"2024-09-14T04:13:24.029519+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028664+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:13:25.028664+0000","last_peered":"2024-09-14T04:13:25.028664+0000","last_clean":"2024-09-14T04:13:25.028664+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:13:25.028664+0000","last_undegraded":"2024-09-14T04:13:25.028664+0000","last_fullsized":"2024-09-14T04:13:25.028664+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"236'14230","reported_seq":14298,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029518+0000","last_change":"2024-09-14T04:08:44.172521+0000","last_active":"2024-09-14T04:13:24.029518+0000","last_peered":"2024-09-14T04:13:24.029518+0000","last_clean":"2024-09-14T04:13:24.029518+0000","last_became_active":"2024-09-14T04:08:44.172051+0000","last_became_peered":"2024-09-14T04:08:44.172051+0000","last_unstale":"2024-09-14T04:13:24.029518+0000","last_undegraded":"2024-09-14T04:13:24.029518+0000","last_fullsized":"2024-09-14T04:13:24.029518+0000","mapping_epoch":184,"log_start":"208'6912","ondisk_log_start":"208'6912","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:12.353009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14230,"num_write_kb":455360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":446,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028276+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:13:24.028276+0000","last_peered":"2024-09-14T04:13:24.028276+0000","last_clean":"2024-09-14T04:13:24.028276+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:13:24.028276+0000","last_undegraded":"2024-09-14T04:13:24.028276+0000","last_fullsized":"2024-09-14T04:13:24.028276+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":175,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029648+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:13:24.029648+0000","last_peered":"2024-09-14T04:13:24.029648+0000","last_clean":"2024-09-14T04:13:24.029648+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:13:24.029648+0000","last_undegraded":"2024-09-14T04:13:24.029648+0000","last_fullsized":"2024-09-14T04:13:24.029648+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":305,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041613+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:13:25.041613+0000","last_peered":"2024-09-14T04:13:25.041613+0000","last_clean":"2024-09-14T04:13:25.041613+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:13:25.041613+0000","last_undegraded":"2024-09-14T04:13:25.041613+0000","last_fullsized":"2024-09-14T04:13:25.041613+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":190,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028077+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:13:24.028077+0000","last_peered":"2024-09-14T04:13:24.028077+0000","last_clean":"2024-09-14T04:13:24.028077+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:13:24.028077+0000","last_undegraded":"2024-09-14T04:13:24.028077+0000","last_fullsized":"2024-09-14T04:13:24.028077+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"236'14272","reported_seq":14340,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156851+0000","last_change":"2024-09-14T04:08:44.228193+0000","last_active":"2024-09-14T04:13:24.156851+0000","last_peered":"2024-09-14T04:13:24.156851+0000","last_clean":"2024-09-14T04:13:24.156851+0000","last_became_active":"2024-09-14T04:08:44.227906+0000","last_became_peered":"2024-09-14T04:08:44.227906+0000","last_unstale":"2024-09-14T04:13:24.156851+0000","last_undegraded":"2024-09-14T04:13:24.156851+0000","last_fullsized":"2024-09-14T04:13:24.156851+0000","mapping_epoch":184,"log_start":"213'7700","ondisk_log_start":"213'7700","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:08:43.148572+0000","last_clean_scrub_stamp":"2024-09-14T04:08:43.148572+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:27:01.262522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14272,"num_write_kb":456704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":530,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.027857+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:13:25.027857+0000","last_peered":"2024-09-14T04:13:25.027857+0000","last_clean":"2024-09-14T04:13:25.027857+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:13:25.027857+0000","last_undegraded":"2024-09-14T04:13:25.027857+0000","last_fullsized":"2024-09-14T04:13:25.027857+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":79,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029138+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:13:25.029138+0000","last_peered":"2024-09-14T04:13:25.029138+0000","last_clean":"2024-09-14T04:13:25.029138+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:13:25.029138+0000","last_undegraded":"2024-09-14T04:13:25.029138+0000","last_fullsized":"2024-09-14T04:13:25.029138+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":239,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156883+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:13:24.156883+0000","last_peered":"2024-09-14T04:13:24.156883+0000","last_clean":"2024-09-14T04:13:24.156883+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:13:24.156883+0000","last_undegraded":"2024-09-14T04:13:24.156883+0000","last_fullsized":"2024-09-14T04:13:24.156883+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":195,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028958+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:13:24.028958+0000","last_peered":"2024-09-14T04:13:24.028958+0000","last_clean":"2024-09-14T04:13:24.028958+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:13:24.028958+0000","last_undegraded":"2024-09-14T04:13:24.028958+0000","last_fullsized":"2024-09-14T04:13:24.028958+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":218,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028455+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:13:24.028455+0000","last_peered":"2024-09-14T04:13:24.028455+0000","last_clean":"2024-09-14T04:13:24.028455+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:13:24.028455+0000","last_undegraded":"2024-09-14T04:13:24.028455+0000","last_fullsized":"2024-09-14T04:13:24.028455+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":218,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028827+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:13:24.028827+0000","last_peered":"2024-09-14T04:13:24.028827+0000","last_clean":"2024-09-14T04:13:24.028827+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:13:24.028827+0000","last_undegraded":"2024-09-14T04:13:24.028827+0000","last_fullsized":"2024-09-14T04:13:24.028827+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041174+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:13:25.041174+0000","last_peered":"2024-09-14T04:13:25.041174+0000","last_clean":"2024-09-14T04:13:25.041174+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:13:25.041174+0000","last_undegraded":"2024-09-14T04:13:25.041174+0000","last_fullsized":"2024-09-14T04:13:25.041174+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":248,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.029309+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:13:24.029309+0000","last_peered":"2024-09-14T04:13:24.029309+0000","last_clean":"2024-09-14T04:13:24.029309+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:13:24.029309+0000","last_undegraded":"2024-09-14T04:13:24.029309+0000","last_fullsized":"2024-09-14T04:13:24.029309+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026614+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:13:24.026614+0000","last_peered":"2024-09-14T04:13:24.026614+0000","last_clean":"2024-09-14T04:13:24.026614+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:13:24.026614+0000","last_undegraded":"2024-09-14T04:13:24.026614+0000","last_fullsized":"2024-09-14T04:13:24.026614+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":172,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028724+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:13:24.028724+0000","last_peered":"2024-09-14T04:13:24.028724+0000","last_clean":"2024-09-14T04:13:24.028724+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:13:24.028724+0000","last_undegraded":"2024-09-14T04:13:24.028724+0000","last_fullsized":"2024-09-14T04:13:24.028724+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028995+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:13:25.028995+0000","last_peered":"2024-09-14T04:13:25.028995+0000","last_clean":"2024-09-14T04:13:25.028995+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:13:25.028995+0000","last_undegraded":"2024-09-14T04:13:25.028995+0000","last_fullsized":"2024-09-14T04:13:25.028995+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":182,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.027195+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:13:25.027195+0000","last_peered":"2024-09-14T04:13:25.027195+0000","last_clean":"2024-09-14T04:13:25.027195+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:13:25.027195+0000","last_undegraded":"2024-09-14T04:13:25.027195+0000","last_fullsized":"2024-09-14T04:13:25.027195+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156811+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:13:24.156811+0000","last_peered":"2024-09-14T04:13:24.156811+0000","last_clean":"2024-09-14T04:13:24.156811+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:13:24.156811+0000","last_undegraded":"2024-09-14T04:13:24.156811+0000","last_fullsized":"2024-09-14T04:13:24.156811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028445+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:13:25.028445+0000","last_peered":"2024-09-14T04:13:25.028445+0000","last_clean":"2024-09-14T04:13:25.028445+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:13:25.028445+0000","last_undegraded":"2024-09-14T04:13:25.028445+0000","last_fullsized":"2024-09-14T04:13:25.028445+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028716+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:13:25.028716+0000","last_peered":"2024-09-14T04:13:25.028716+0000","last_clean":"2024-09-14T04:13:25.028716+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:13:25.028716+0000","last_undegraded":"2024-09-14T04:13:25.028716+0000","last_fullsized":"2024-09-14T04:13:25.028716+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":331,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.041070+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:13:25.041070+0000","last_peered":"2024-09-14T04:13:25.041070+0000","last_clean":"2024-09-14T04:13:25.041070+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:13:25.041070+0000","last_undegraded":"2024-09-14T04:13:25.041070+0000","last_fullsized":"2024-09-14T04:13:25.041070+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":161,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026199+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:13:24.026199+0000","last_peered":"2024-09-14T04:13:24.026199+0000","last_clean":"2024-09-14T04:13:24.026199+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:13:24.026199+0000","last_undegraded":"2024-09-14T04:13:24.026199+0000","last_fullsized":"2024-09-14T04:13:24.026199+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":229,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029453+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:13:25.029453+0000","last_peered":"2024-09-14T04:13:25.029453+0000","last_clean":"2024-09-14T04:13:25.029453+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:13:25.029453+0000","last_undegraded":"2024-09-14T04:13:25.029453+0000","last_fullsized":"2024-09-14T04:13:25.029453+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.030154+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:13:25.030154+0000","last_peered":"2024-09-14T04:13:25.030154+0000","last_clean":"2024-09-14T04:13:25.030154+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:13:25.030154+0000","last_undegraded":"2024-09-14T04:13:25.030154+0000","last_fullsized":"2024-09-14T04:13:25.030154+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":75,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.028496+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:13:25.028496+0000","last_peered":"2024-09-14T04:13:25.028496+0000","last_clean":"2024-09-14T04:13:25.028496+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:13:25.028496+0000","last_undegraded":"2024-09-14T04:13:25.028496+0000","last_fullsized":"2024-09-14T04:13:25.028496+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028269+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:13:24.028269+0000","last_peered":"2024-09-14T04:13:24.028269+0000","last_clean":"2024-09-14T04:13:24.028269+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:13:24.028269+0000","last_undegraded":"2024-09-14T04:13:24.028269+0000","last_fullsized":"2024-09-14T04:13:24.028269+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":248,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026404+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:13:24.026404+0000","last_peered":"2024-09-14T04:13:24.026404+0000","last_clean":"2024-09-14T04:13:24.026404+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:13:24.026404+0000","last_undegraded":"2024-09-14T04:13:24.026404+0000","last_fullsized":"2024-09-14T04:13:24.026404+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028389+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:13:24.028389+0000","last_peered":"2024-09-14T04:13:24.028389+0000","last_clean":"2024-09-14T04:13:24.028389+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:13:24.028389+0000","last_undegraded":"2024-09-14T04:13:24.028389+0000","last_fullsized":"2024-09-14T04:13:24.028389+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":235,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.026317+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:13:24.026317+0000","last_peered":"2024-09-14T04:13:24.026317+0000","last_clean":"2024-09-14T04:13:24.026317+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:13:24.026317+0000","last_undegraded":"2024-09-14T04:13:24.026317+0000","last_fullsized":"2024-09-14T04:13:24.026317+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":309,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.030067+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:13:25.030067+0000","last_peered":"2024-09-14T04:13:25.030067+0000","last_clean":"2024-09-14T04:13:25.030067+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:13:25.030067+0000","last_undegraded":"2024-09-14T04:13:25.030067+0000","last_fullsized":"2024-09-14T04:13:25.030067+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":117,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.156766+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:13:24.156766+0000","last_peered":"2024-09-14T04:13:24.156766+0000","last_clean":"2024-09-14T04:13:24.156766+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:13:24.156766+0000","last_undegraded":"2024-09-14T04:13:24.156766+0000","last_fullsized":"2024-09-14T04:13:24.156766+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-09-14T04:13:24.028061+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:13:24.028061+0000","last_peered":"2024-09-14T04:13:24.028061+0000","last_clean":"2024-09-14T04:13:24.028061+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:13:24.028061+0000","last_undegraded":"2024-09-14T04:13:24.028061+0000","last_fullsized":"2024-09-14T04:13:24.028061+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":81,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-09-14T04:13:25.029044+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:13:25.029044+0000","last_peered":"2024-09-14T04:13:25.029044+0000","last_clean":"2024-09-14T04:13:25.029044+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:13:25.029044+0000","last_undegraded":"2024-09-14T04:13:25.029044+0000","last_fullsized":"2024-09-14T04:13:25.029044+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5446,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":230276,"num_write_kb":7368769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23827,"num_bytes_recovered":819068928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":336658432,"data_stored":336658432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120033,"ondisk_log_size":120033,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201676,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":800900,"kb_used_data":3576,"kb_used_omap":1,"kb_used_meta":797310,"kb_avail":92944252,"statfs":{"total":95995035648,"available":95174914048,"internally_reserved":0,"allocated":3661824,"data_stored":2118340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":816445899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463314,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661944,"kb_used_data":3244,"kb_used_omap":1,"kb_used_meta":658686,"kb_avail":93083208,"statfs":{"total":95995035648,"available":95317204992,"internally_reserved":0,"allocated":3321856,"data_stored":1790013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":674494919},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724950,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027580,"kb_used_data":3820,"kb_used_omap":1,"kb_used_meta":1023742,"kb_avail":92717572,"statfs":{"total":95995035648,"available":94942793728,"internally_reserved":0,"allocated":3911680,"data_stored":2376480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048312262},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986586,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":832576,"kb_used_data":3252,"kb_used_omap":1,"kb_used_meta":829310,"kb_avail":92912576,"statfs":{"total":95995035648,"available":95142477824,"internally_reserved":0,"allocated":3330048,"data_stored":1790351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":849213894},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748615,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412084,"kb_used_data":3816,"kb_used_omap":4,"kb_used_meta":408251,"kb_avail":93333068,"statfs":{"total":95995035648,"available":95573061632,"internally_reserved":0,"allocated":3907584,"data_stored":2376192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":418049580},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912141,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601588,"kb_used_data":334188,"kb_used_omap":6,"kb_used_meta":267385,"kb_avail":93143564,"statfs":{"total":95995035648,"available":95379009536,"internally_reserved":0,"allocated":342208512,"data_stored":340673550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":273802809},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468158,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494704,"kb_used_data":3232,"kb_used_omap":5,"kb_used_meta":491450,"kb_avail":93250448,"statfs":{"total":95995035648,"available":95488458752,"internally_reserved":0,"allocated":3309568,"data_stored":1784333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":503245151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342273,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219188,"kb_used_data":3244,"kb_used_omap":10,"kb_used_meta":215925,"kb_avail":93525964,"statfs":{"total":95995035648,"available":95770587136,"internally_reserved":0,"allocated":3321856,"data_stored":1786372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":221107410},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":336658432,"data_stored":336658432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:13:40.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:13:40.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:40.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:40.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:40.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:40.354 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:40 smithi001 ceph-mon[112165]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:40.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:40 smithi001 ceph-mon[112165]: pgmap v844: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 71 objects/s recovering 2024-09-14T04:13:40.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:40 smithi001 ceph-mon[107399]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:40.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:40 smithi001 ceph-mon[107399]: pgmap v844: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 71 objects/s recovering 2024-09-14T04:13:40.420 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:40.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:40.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:40.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:40 smithi012 ceph-mon[101180]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:13:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:40 smithi012 ceph-mon[101180]: pgmap v844: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5446/78 objects misplaced (6982.051%); 0 B/s, 71 objects/s recovering 2024-09-14T04:13:40.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:40.695 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:40.720 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:40.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:40.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:40.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:40.949 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:40.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:41.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:41.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:41.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:41.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:41.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:41.278 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_5' removed 2024-09-14T04:13:41.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:41.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:41.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:41.382 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2038328200' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.382 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2038328200' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.382 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:41.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:41.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:41.599 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2038328200' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.600 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:13:41.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:41.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:41.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:41.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:41.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:41.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:41.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:41.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:42.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:42.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:42.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:42.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:42.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:42.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:42.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:42.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:42.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:42 smithi012 ceph-mon[101180]: pgmap v845: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 4987/78 objects misplaced (6393.590%); 0 B/s, 109 objects/s recovering 2024-09-14T04:13:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:13:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:42 smithi012 ceph-mon[101180]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T04:13:42.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[112165]: pgmap v845: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 4987/78 objects misplaced (6393.590%); 0 B/s, 109 objects/s recovering 2024-09-14T04:13:42.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:13:42.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[112165]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T04:13:42.599 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[107399]: pgmap v845: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 4987/78 objects misplaced (6393.590%); 0 B/s, 109 objects/s recovering 2024-09-14T04:13:42.599 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:13:42.600 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:42 smithi001 ceph-mon[107399]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T04:13:42.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:42.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:42.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:42.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:42.867 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:42.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:42.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:42.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:42.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:43.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:43.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:43.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:43.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:43.333 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:43.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:43.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:43.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:43.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:43.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:43.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:43.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:43.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:43.835 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:13:43.836 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:13:43.836 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-09-14T04:13:43.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_6 16 2024-09-14T04:13:43.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:43.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:43.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:44.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:44.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:44.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:44.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:44.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:44.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:44.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:44.340 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:44 smithi001 ceph-mon[107399]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:44.340 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:44 smithi001 ceph-mon[112165]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:44.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:44.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:44.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:44.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:44 smithi012 ceph-mon[101180]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:44.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:44.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:44.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:44.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:44.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:44.919 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:44.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:45.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:45.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:45.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:45.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:45.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:45.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:45.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:45.432 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:45.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:45.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2148687941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:45.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:45.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2148687941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:45.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2148687941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:13:45.686 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:45.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:45.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:45.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:45.943 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:45.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:46.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:46.071 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:46.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:46.104 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:13:46.104 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:13:46.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:46.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:46.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:46.323 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:46.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:46.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:13:46.451 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:13:46.452 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:13:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:46.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:46.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:46.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:46 smithi012 ceph-mon[101180]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:46.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:46.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:46 smithi001 ceph-mon[107399]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:46.764 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:46 smithi001 ceph-mon[112165]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:46.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:46.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:46.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:46.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:46] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-14T04:13:46.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:47.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:47.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:47.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:47.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:47.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.276 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:47.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:47.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:47.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:47.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:47.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:47.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:47.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:47.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:47.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:47.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:47.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:48.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:48.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:48.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:48.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:13:48.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:48.169 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:48.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:48.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:48.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:48.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:48.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:48.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:48.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:48.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:48 smithi012 ceph-mon[101180]: pgmap v849: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:48.616 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:48 smithi001 ceph-mon[107399]: pgmap v849: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:48.617 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:48 smithi001 ceph-mon[112165]: pgmap v849: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:48.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:48.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:48.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:48.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:48.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:48.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:48.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:49.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:49.054 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:49.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:49.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:13:49.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:49.283 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:49.306 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:49.356 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:13:49.356 INFO:teuthology.orchestra.run.smithi001.stdout:{"fsid":"5626d550-724a-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1250,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":242,"num_osds":8,"num_up_osds":8,"osd_up_since":1726286894,"num_in_osds":8,"osd_in_since":1726286805,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83}],"num_pgs":83,"num_pools":4,"num_objects":26,"data_bytes":595941,"bytes_used":5185667072,"bytes_avail":762774618112,"bytes_total":767960285184},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi001.ytjgbj","status":"up:active","gid":24557}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-14T04:09:15.106156+0000","services":{}},"progress_events":{}} 2024-09-14T04:13:49.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:49.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:49.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:49.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:49.564 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:49.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:49.683 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:49.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:49.806 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:49.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:49.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:49.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:50.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:50.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:50.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:50.184 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-09-14T04:13:50.185 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:13:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.187 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:50.233 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:50.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:50.294 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:13:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:50.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:50.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:50.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[107399]: pgmap v850: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2195954650' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/468834547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[112165]: pgmap v850: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2195954650' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:13:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/468834547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:50 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:50.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:50 smithi012 ceph-mon[101180]: pgmap v850: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2195954650' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:13:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/468834547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:50 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:13:50.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:50.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:50.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:50.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:50.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:51.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:51.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:51.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:51.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:51.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:51.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:51.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:51.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:51.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:51.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:51.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:51.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:13:51.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:13:51.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:13:51.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[107399]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[112165]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T04:13:51.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:51 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:13:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:51 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:13:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:51 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:13:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:51 smithi012 ceph-mon[101180]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T04:13:51.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:51.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:51.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:51.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:51.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:51.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:51.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:51.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:52.042 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:52.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:52.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:52.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:52.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:52.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:52.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:52.437 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:52.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:52 smithi001 ceph-mon[107399]: pgmap v852: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:52.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:52 smithi001 ceph-mon[112165]: pgmap v852: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:52.538 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:52.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:52 smithi012 ceph-mon[101180]: pgmap v852: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:52.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:52.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:52.788 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:52.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:52.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:52.934 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:52.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:52.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:53.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:53.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:53.160 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:53.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:53.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:53.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:53.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:53.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:53.419 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:13:53.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:53.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:53.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:53.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:53.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:53.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:53.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:53.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:53.936 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:53.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:54.002 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:54.038 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:54.062 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:54.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:54.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:54.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:54.309 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_6' created 2024-09-14T04:13:54.312 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:54.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[112165]: pgmap v853: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2215669369' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[107399]: pgmap v853: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2215669369' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.436 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:54 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.437 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:54.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:54.551 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:13:54.551 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:13:54.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:54 smithi012 ceph-mon[101180]: pgmap v853: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2215669369' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:54 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-14T04:13:54.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:54.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:54.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:54.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:54.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:54.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:54.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:55.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:55.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.059 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:55.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:55.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:55.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:55.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:55.308 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:55.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-14T04:13:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:55 smithi012 ceph-mon[101180]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T04:13:55.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:55.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:55.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:55.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:55.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:55.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:55.682 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:55.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-14T04:13:55.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:55 smithi001 ceph-mon[112165]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T04:13:55.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-14T04:13:55.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:55 smithi001 ceph-mon[107399]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T04:13:55.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-09-14T04:13:55.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:55.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:55.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:55.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:55.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:56.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:56.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:56.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:56.179 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:56.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:56.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:13:56.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:13:56.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:13:56.407 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:56.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:56.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:56.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:56 smithi012 ceph-mon[101180]: pgmap v855: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:56 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:13:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:56 smithi012 ceph-mon[101180]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T04:13:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[107399]: pgmap v855: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[107399]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[112165]: pgmap v855: 99 pgs: 2 creating+peering, 14 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:56.651 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:13:56.652 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[112165]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T04:13:56.652 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.652 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:13:56.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:13:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:56.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:13:56.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:56.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:56.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:56.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:56.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:56.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:13:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:13:56] "GET /metrics HTTP/1.1" 200 34162 "" "Prometheus/2.51.0" 2024-09-14T04:13:57.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:57.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:57.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:57.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:57.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:57.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:57.326 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:57.345 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:57.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:57.448 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:57 smithi001 ceph-mon[112165]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T04:13:57.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:57 smithi001 ceph-mon[107399]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T04:13:57.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:57.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:57.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:57 smithi012 ceph-mon[101180]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T04:13:57.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:57.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:57.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:57.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:57.881 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:57.891 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:13:57.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:57.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:57.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:58.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:58.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:58.186 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:58.208 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:58.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:58.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:58.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:58.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:58.450 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:13:58 smithi001 ceph-mon[107399]: pgmap v858: 99 pgs: 2 creating+peering, 3 unknown, 94 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:58.451 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:13:58 smithi001 ceph-mon[112165]: pgmap v858: 99 pgs: 2 creating+peering, 3 unknown, 94 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:58.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:58.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:58.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:13:58.588 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:13:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:13:58 smithi012 ceph-mon[101180]: pgmap v858: 99 pgs: 2 creating+peering, 3 unknown, 94 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:13:58.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:58.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:58.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:58.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:13:58.838 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:13:58.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:13:58.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:58.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:13:59.068 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:59.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:59.194 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:59.217 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:59.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:59.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:13:59.447 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:59.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:59.529 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:13:59.529 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:13:59.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:59.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:59.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:13:59.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:13:59.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:13:59.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:13:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:13:59.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:13:59.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:13:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:13:59.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:00.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:00.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:00.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:00.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:00.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:00.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:00.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:00.457 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:00.479 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:00.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:00.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:00 smithi012 ceph-mon[101180]: pgmap v859: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:00.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:00.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:00 smithi001 ceph-mon[107399]: pgmap v859: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:00 smithi001 ceph-mon[112165]: pgmap v859: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:00.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:00.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:00.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:00.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:00.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:00.990 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:00.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:01.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:01.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:01.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:01.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:01.249 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:01.351 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:01.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:01.476 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:01.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:01.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:01 smithi012 ceph-mon[101180]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:01.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:01.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:01 smithi001 ceph-mon[107399]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:01 smithi001 ceph-mon[112165]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:01.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:01.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:01.811 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":859,"stamp":"2024-09-14T04:13:59.214215+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":918994,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5063288,"kb_used_data":312784,"kb_used_omap":33,"kb_used_meta":4750366,"kb_avail":744897928,"statfs":{"total":767960285184,"available":762775478272,"internally_reserved":0,"allocated":320290816,"data_stored":307855074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4864375220},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001327"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":282,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310848+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:13:56.310848+0000","last_peered":"2024-09-14T04:13:56.310848+0000","last_clean":"2024-09-14T04:13:56.310848+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:13:56.310848+0000","last_undegraded":"2024-09-14T04:13:56.310848+0000","last_fullsized":"2024-09-14T04:13:56.310848+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":235,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312512+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:13:56.312512+0000","last_peered":"2024-09-14T04:13:56.312512+0000","last_clean":"2024-09-14T04:13:56.312512+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:13:56.312512+0000","last_undegraded":"2024-09-14T04:13:56.312512+0000","last_fullsized":"2024-09-14T04:13:56.312512+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":55,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300971+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:13:54.300971+0000","last_peered":"2024-09-14T04:13:54.300971+0000","last_clean":"2024-09-14T04:13:54.300971+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:13:54.300971+0000","last_undegraded":"2024-09-14T04:13:54.300971+0000","last_fullsized":"2024-09-14T04:13:54.300971+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.311601+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:13:56.311601+0000","last_peered":"2024-09-14T04:13:56.311601+0000","last_clean":"2024-09-14T04:13:56.311601+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:13:56.311601+0000","last_undegraded":"2024-09-14T04:13:56.311601+0000","last_fullsized":"2024-09-14T04:13:56.311601+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312406+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:13:56.312406+0000","last_peered":"2024-09-14T04:13:56.312406+0000","last_clean":"2024-09-14T04:13:56.312406+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:13:56.312406+0000","last_undegraded":"2024-09-14T04:13:56.312406+0000","last_fullsized":"2024-09-14T04:13:56.312406+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303281+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:13:54.303281+0000","last_peered":"2024-09-14T04:13:54.303281+0000","last_clean":"2024-09-14T04:13:54.303281+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:13:54.303281+0000","last_undegraded":"2024-09-14T04:13:54.303281+0000","last_fullsized":"2024-09-14T04:13:54.303281+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.309470+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:13:55.309470+0000","last_peered":"2024-09-14T04:13:55.309470+0000","last_clean":"2024-09-14T04:13:55.309470+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:13:55.309470+0000","last_undegraded":"2024-09-14T04:13:55.309470+0000","last_fullsized":"2024-09-14T04:13:55.309470+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":296,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310064+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:13:54.310064+0000","last_peered":"2024-09-14T04:13:54.310064+0000","last_clean":"2024-09-14T04:13:54.310064+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:13:54.310064+0000","last_undegraded":"2024-09-14T04:13:54.310064+0000","last_fullsized":"2024-09-14T04:13:54.310064+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.311357+0000","last_change":"2024-09-14T04:13:55.311089+0000","last_active":"2024-09-14T04:13:55.311357+0000","last_peered":"2024-09-14T04:13:55.311357+0000","last_clean":"2024-09-14T04:13:55.311357+0000","last_became_active":"2024-09-14T04:13:55.310788+0000","last_became_peered":"2024-09-14T04:13:55.310788+0000","last_unstale":"2024-09-14T04:13:55.311357+0000","last_undegraded":"2024-09-14T04:13:55.311357+0000","last_fullsized":"2024-09-14T04:13:55.311357+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:55.525731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":256,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303896+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:13:54.303896+0000","last_peered":"2024-09-14T04:13:54.303896+0000","last_clean":"2024-09-14T04:13:54.303896+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:13:54.303896+0000","last_undegraded":"2024-09-14T04:13:54.303896+0000","last_fullsized":"2024-09-14T04:13:54.303896+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300650+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:13:54.300650+0000","last_peered":"2024-09-14T04:13:54.300650+0000","last_clean":"2024-09-14T04:13:54.300650+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:13:54.300650+0000","last_undegraded":"2024-09-14T04:13:54.300650+0000","last_fullsized":"2024-09-14T04:13:54.300650+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":14,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312305+0000","last_change":"2024-09-14T04:13:55.317315+0000","last_active":"2024-09-14T04:13:56.312305+0000","last_peered":"2024-09-14T04:13:56.312305+0000","last_clean":"2024-09-14T04:13:56.312305+0000","last_became_active":"2024-09-14T04:13:55.316972+0000","last_became_peered":"2024-09-14T04:13:55.316972+0000","last_unstale":"2024-09-14T04:13:56.312305+0000","last_undegraded":"2024-09-14T04:13:56.312305+0000","last_fullsized":"2024-09-14T04:13:56.312305+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:18.321602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":255,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303843+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:13:54.303843+0000","last_peered":"2024-09-14T04:13:54.303843+0000","last_clean":"2024-09-14T04:13:54.303843+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:13:54.303843+0000","last_undegraded":"2024-09-14T04:13:54.303843+0000","last_fullsized":"2024-09-14T04:13:54.303843+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301960+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:13:54.301960+0000","last_peered":"2024-09-14T04:13:54.301960+0000","last_clean":"2024-09-14T04:13:54.301960+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:13:54.301960+0000","last_undegraded":"2024-09-14T04:13:54.301960+0000","last_fullsized":"2024-09-14T04:13:54.301960+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.316421+0000","last_change":"2024-09-14T04:13:55.316081+0000","last_active":"2024-09-14T04:13:55.316421+0000","last_peered":"2024-09-14T04:13:55.316421+0000","last_clean":"2024-09-14T04:13:55.316421+0000","last_became_active":"2024-09-14T04:13:55.315775+0000","last_became_peered":"2024-09-14T04:13:55.315775+0000","last_unstale":"2024-09-14T04:13:55.316421+0000","last_undegraded":"2024-09-14T04:13:55.316421+0000","last_fullsized":"2024-09-14T04:13:55.316421+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:39:05.829936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":226,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303646+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:13:54.303646+0000","last_peered":"2024-09-14T04:13:54.303646+0000","last_clean":"2024-09-14T04:13:54.303646+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:13:54.303646+0000","last_undegraded":"2024-09-14T04:13:54.303646+0000","last_fullsized":"2024-09-14T04:13:54.303646+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":212,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310132+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:13:54.310132+0000","last_peered":"2024-09-14T04:13:54.310132+0000","last_clean":"2024-09-14T04:13:54.310132+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:13:54.310132+0000","last_undegraded":"2024-09-14T04:13:54.310132+0000","last_fullsized":"2024-09-14T04:13:54.310132+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.315714+0000","last_change":"2024-09-14T04:13:55.315421+0000","last_active":"2024-09-14T04:13:55.315714+0000","last_peered":"2024-09-14T04:13:55.315714+0000","last_clean":"2024-09-14T04:13:55.315714+0000","last_became_active":"2024-09-14T04:13:55.315144+0000","last_became_peered":"2024-09-14T04:13:55.315144+0000","last_unstale":"2024-09-14T04:13:55.315714+0000","last_undegraded":"2024-09-14T04:13:55.315714+0000","last_fullsized":"2024-09-14T04:13:55.315714+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:17.194435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300872+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:13:54.300872+0000","last_peered":"2024-09-14T04:13:54.300872+0000","last_clean":"2024-09-14T04:13:54.300872+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:13:54.300872+0000","last_undegraded":"2024-09-14T04:13:54.300872+0000","last_fullsized":"2024-09-14T04:13:54.300872+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312629+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:13:56.312629+0000","last_peered":"2024-09-14T04:13:56.312629+0000","last_clean":"2024-09-14T04:13:56.312629+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:13:56.312629+0000","last_undegraded":"2024-09-14T04:13:56.312629+0000","last_fullsized":"2024-09-14T04:13:56.312629+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.316211+0000","last_change":"2024-09-14T04:13:55.315439+0000","last_active":"2024-09-14T04:13:55.316211+0000","last_peered":"2024-09-14T04:13:55.316211+0000","last_clean":"2024-09-14T04:13:55.316211+0000","last_became_active":"2024-09-14T04:13:55.315149+0000","last_became_peered":"2024-09-14T04:13:55.315149+0000","last_unstale":"2024-09-14T04:13:55.316211+0000","last_undegraded":"2024-09-14T04:13:55.316211+0000","last_fullsized":"2024-09-14T04:13:55.316211+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:40.970593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":597,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303145+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:13:54.303145+0000","last_peered":"2024-09-14T04:13:54.303145+0000","last_clean":"2024-09-14T04:13:54.303145+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:13:54.303145+0000","last_undegraded":"2024-09-14T04:13:54.303145+0000","last_fullsized":"2024-09-14T04:13:54.303145+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":89,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312292+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:13:56.312292+0000","last_peered":"2024-09-14T04:13:56.312292+0000","last_clean":"2024-09-14T04:13:56.312292+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:13:56.312292+0000","last_undegraded":"2024-09-14T04:13:56.312292+0000","last_fullsized":"2024-09-14T04:13:56.312292+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":231,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302433+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:13:54.302433+0000","last_peered":"2024-09-14T04:13:54.302433+0000","last_clean":"2024-09-14T04:13:54.302433+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:13:54.302433+0000","last_undegraded":"2024-09-14T04:13:54.302433+0000","last_fullsized":"2024-09-14T04:13:54.302433+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":14,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310748+0000","last_change":"2024-09-14T04:13:55.315267+0000","last_active":"2024-09-14T04:13:56.310748+0000","last_peered":"2024-09-14T04:13:56.310748+0000","last_clean":"2024-09-14T04:13:56.310748+0000","last_became_active":"2024-09-14T04:13:55.314973+0000","last_became_peered":"2024-09-14T04:13:55.314973+0000","last_unstale":"2024-09-14T04:13:56.310748+0000","last_undegraded":"2024-09-14T04:13:56.310748+0000","last_fullsized":"2024-09-14T04:13:56.310748+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:40:38.432894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":466,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300586+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:13:54.300586+0000","last_peered":"2024-09-14T04:13:54.300586+0000","last_clean":"2024-09-14T04:13:54.300586+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:13:54.300586+0000","last_undegraded":"2024-09-14T04:13:54.300586+0000","last_fullsized":"2024-09-14T04:13:54.300586+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310233+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:13:54.310233+0000","last_peered":"2024-09-14T04:13:54.310233+0000","last_clean":"2024-09-14T04:13:54.310233+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:13:54.310233+0000","last_undegraded":"2024-09-14T04:13:54.310233+0000","last_fullsized":"2024-09-14T04:13:54.310233+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":36,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302659+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:13:54.302659+0000","last_peered":"2024-09-14T04:13:54.302659+0000","last_clean":"2024-09-14T04:13:54.302659+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:13:54.302659+0000","last_undegraded":"2024-09-14T04:13:54.302659+0000","last_fullsized":"2024-09-14T04:13:54.302659+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.316189+0000","last_change":"2024-09-14T04:13:55.315758+0000","last_active":"2024-09-14T04:13:55.316189+0000","last_peered":"2024-09-14T04:13:55.316189+0000","last_clean":"2024-09-14T04:13:55.316189+0000","last_became_active":"2024-09-14T04:13:55.314723+0000","last_became_peered":"2024-09-14T04:13:55.314723+0000","last_unstale":"2024-09-14T04:13:55.316189+0000","last_undegraded":"2024-09-14T04:13:55.316189+0000","last_fullsized":"2024-09-14T04:13:55.316189+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:53.745702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":36,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301882+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:13:54.301882+0000","last_peered":"2024-09-14T04:13:54.301882+0000","last_clean":"2024-09-14T04:13:54.301882+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:13:54.301882+0000","last_undegraded":"2024-09-14T04:13:54.301882+0000","last_fullsized":"2024-09-14T04:13:54.301882+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302890+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:13:54.302890+0000","last_peered":"2024-09-14T04:13:54.302890+0000","last_clean":"2024-09-14T04:13:54.302890+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:13:54.302890+0000","last_undegraded":"2024-09-14T04:13:54.302890+0000","last_fullsized":"2024-09-14T04:13:54.302890+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":1,"reported_epoch":244,"state":"creating+peering","last_fresh":"2024-09-14T04:13:54.303364+0000","last_change":"2024-09-14T04:13:54.303364+0000","last_active":"2024-09-14T04:13:54.293151+0000","last_peered":"2024-09-14T04:13:54.293151+0000","last_clean":"2024-09-14T04:13:54.293151+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-14T04:13:54.303364+0000","last_undegraded":"2024-09-14T04:13:54.303364+0000","last_fullsized":"2024-09-14T04:13:54.303364+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":450,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300811+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:13:54.300811+0000","last_peered":"2024-09-14T04:13:54.300811+0000","last_clean":"2024-09-14T04:13:54.300811+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:13:54.300811+0000","last_undegraded":"2024-09-14T04:13:54.300811+0000","last_fullsized":"2024-09-14T04:13:54.300811+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.308555+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:13:55.308555+0000","last_peered":"2024-09-14T04:13:55.308555+0000","last_clean":"2024-09-14T04:13:55.308555+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:13:55.308555+0000","last_undegraded":"2024-09-14T04:13:55.308555+0000","last_fullsized":"2024-09-14T04:13:55.308555+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":325,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302964+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:13:54.302964+0000","last_peered":"2024-09-14T04:13:54.302964+0000","last_clean":"2024-09-14T04:13:54.302964+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:13:54.302964+0000","last_undegraded":"2024-09-14T04:13:54.302964+0000","last_fullsized":"2024-09-14T04:13:54.302964+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.315970+0000","last_change":"2024-09-14T04:13:55.314904+0000","last_active":"2024-09-14T04:13:55.315970+0000","last_peered":"2024-09-14T04:13:55.315970+0000","last_clean":"2024-09-14T04:13:55.315970+0000","last_became_active":"2024-09-14T04:13:55.314584+0000","last_became_peered":"2024-09-14T04:13:55.314584+0000","last_unstale":"2024-09-14T04:13:55.315970+0000","last_undegraded":"2024-09-14T04:13:55.315970+0000","last_fullsized":"2024-09-14T04:13:55.315970+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:15:01.720889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":615,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303271+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:13:54.303271+0000","last_peered":"2024-09-14T04:13:54.303271+0000","last_clean":"2024-09-14T04:13:54.303271+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:13:54.303271+0000","last_undegraded":"2024-09-14T04:13:54.303271+0000","last_fullsized":"2024-09-14T04:13:54.303271+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":229,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300649+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:13:54.300649+0000","last_peered":"2024-09-14T04:13:54.300649+0000","last_clean":"2024-09-14T04:13:54.300649+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:13:54.300649+0000","last_undegraded":"2024-09-14T04:13:54.300649+0000","last_fullsized":"2024-09-14T04:13:54.300649+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300838+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:13:54.300838+0000","last_peered":"2024-09-14T04:13:54.300838+0000","last_clean":"2024-09-14T04:13:54.300838+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:13:54.300838+0000","last_undegraded":"2024-09-14T04:13:54.300838+0000","last_fullsized":"2024-09-14T04:13:54.300838+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310484+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:13:54.310484+0000","last_peered":"2024-09-14T04:13:54.310484+0000","last_clean":"2024-09-14T04:13:54.310484+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:13:54.310484+0000","last_undegraded":"2024-09-14T04:13:54.310484+0000","last_fullsized":"2024-09-14T04:13:54.310484+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":11,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.314423+0000","last_change":"2024-09-14T04:13:55.313345+0000","last_active":"2024-09-14T04:13:55.314423+0000","last_peered":"2024-09-14T04:13:55.314423+0000","last_clean":"2024-09-14T04:13:55.314423+0000","last_became_active":"2024-09-14T04:13:55.313088+0000","last_became_peered":"2024-09-14T04:13:55.313088+0000","last_unstale":"2024-09-14T04:13:55.314423+0000","last_undegraded":"2024-09-14T04:13:55.314423+0000","last_fullsized":"2024-09-14T04:13:55.314423+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:56.220996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":508,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.304706+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:13:54.304706+0000","last_peered":"2024-09-14T04:13:54.304706+0000","last_clean":"2024-09-14T04:13:54.304706+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:13:54.304706+0000","last_undegraded":"2024-09-14T04:13:54.304706+0000","last_fullsized":"2024-09-14T04:13:54.304706+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":259,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303183+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:13:54.303183+0000","last_peered":"2024-09-14T04:13:54.303183+0000","last_clean":"2024-09-14T04:13:54.303183+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:13:54.303183+0000","last_undegraded":"2024-09-14T04:13:54.303183+0000","last_fullsized":"2024-09-14T04:13:54.303183+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":308,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.311665+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:13:56.311665+0000","last_peered":"2024-09-14T04:13:56.311665+0000","last_clean":"2024-09-14T04:13:56.311665+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:13:56.311665+0000","last_undegraded":"2024-09-14T04:13:56.311665+0000","last_fullsized":"2024-09-14T04:13:56.311665+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312487+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:13:56.312487+0000","last_peered":"2024-09-14T04:13:56.312487+0000","last_clean":"2024-09-14T04:13:56.312487+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:13:56.312487+0000","last_undegraded":"2024-09-14T04:13:56.312487+0000","last_fullsized":"2024-09-14T04:13:56.312487+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.315984+0000","last_change":"2024-09-14T04:13:55.314961+0000","last_active":"2024-09-14T04:13:55.315984+0000","last_peered":"2024-09-14T04:13:55.315984+0000","last_clean":"2024-09-14T04:13:55.315984+0000","last_became_active":"2024-09-14T04:13:55.314601+0000","last_became_peered":"2024-09-14T04:13:55.314601+0000","last_unstale":"2024-09-14T04:13:55.315984+0000","last_undegraded":"2024-09-14T04:13:55.315984+0000","last_fullsized":"2024-09-14T04:13:55.315984+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:32.900254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":477,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303145+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:13:54.303145+0000","last_peered":"2024-09-14T04:13:54.303145+0000","last_clean":"2024-09-14T04:13:54.303145+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:13:54.303145+0000","last_undegraded":"2024-09-14T04:13:54.303145+0000","last_fullsized":"2024-09-14T04:13:54.303145+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":371,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310416+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:13:56.310416+0000","last_peered":"2024-09-14T04:13:56.310416+0000","last_clean":"2024-09-14T04:13:56.310416+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:13:56.310416+0000","last_undegraded":"2024-09-14T04:13:56.310416+0000","last_fullsized":"2024-09-14T04:13:56.310416+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303959+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:13:54.303959+0000","last_peered":"2024-09-14T04:13:54.303959+0000","last_clean":"2024-09-14T04:13:54.303959+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:13:54.303959+0000","last_undegraded":"2024-09-14T04:13:54.303959+0000","last_fullsized":"2024-09-14T04:13:54.303959+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":379,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302764+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:13:54.302764+0000","last_peered":"2024-09-14T04:13:54.302764+0000","last_clean":"2024-09-14T04:13:54.302764+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:13:54.302764+0000","last_undegraded":"2024-09-14T04:13:54.302764+0000","last_fullsized":"2024-09-14T04:13:54.302764+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":11,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.314473+0000","last_change":"2024-09-14T04:13:55.313340+0000","last_active":"2024-09-14T04:13:55.314473+0000","last_peered":"2024-09-14T04:13:55.314473+0000","last_clean":"2024-09-14T04:13:55.314473+0000","last_became_active":"2024-09-14T04:13:55.313042+0000","last_became_peered":"2024-09-14T04:13:55.313042+0000","last_unstale":"2024-09-14T04:13:55.314473+0000","last_undegraded":"2024-09-14T04:13:55.314473+0000","last_fullsized":"2024-09-14T04:13:55.314473+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:03:15.019731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":484,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300586+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:13:54.300586+0000","last_peered":"2024-09-14T04:13:54.300586+0000","last_clean":"2024-09-14T04:13:54.300586+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:13:54.300586+0000","last_undegraded":"2024-09-14T04:13:54.300586+0000","last_fullsized":"2024-09-14T04:13:54.300586+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":230,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300909+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:13:54.300909+0000","last_peered":"2024-09-14T04:13:54.300909+0000","last_clean":"2024-09-14T04:13:54.300909+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:13:54.300909+0000","last_undegraded":"2024-09-14T04:13:54.300909+0000","last_fullsized":"2024-09-14T04:13:54.300909+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":249,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.306608+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:13:55.306608+0000","last_peered":"2024-09-14T04:13:55.306608+0000","last_clean":"2024-09-14T04:13:55.306608+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:13:55.306608+0000","last_undegraded":"2024-09-14T04:13:55.306608+0000","last_fullsized":"2024-09-14T04:13:55.306608+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300662+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:13:54.300662+0000","last_peered":"2024-09-14T04:13:54.300662+0000","last_clean":"2024-09-14T04:13:54.300662+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:13:54.300662+0000","last_undegraded":"2024-09-14T04:13:54.300662+0000","last_fullsized":"2024-09-14T04:13:54.300662+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":14,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310467+0000","last_change":"2024-09-14T04:13:55.314198+0000","last_active":"2024-09-14T04:13:56.310467+0000","last_peered":"2024-09-14T04:13:56.310467+0000","last_clean":"2024-09-14T04:13:56.310467+0000","last_became_active":"2024-09-14T04:13:55.313909+0000","last_became_peered":"2024-09-14T04:13:55.313909+0000","last_unstale":"2024-09-14T04:13:56.310467+0000","last_undegraded":"2024-09-14T04:13:56.310467+0000","last_fullsized":"2024-09-14T04:13:56.310467+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:21.447314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":458,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310531+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:13:54.310531+0000","last_peered":"2024-09-14T04:13:54.310531+0000","last_clean":"2024-09-14T04:13:54.310531+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:13:54.310531+0000","last_undegraded":"2024-09-14T04:13:54.310531+0000","last_fullsized":"2024-09-14T04:13:54.310531+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":253,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301052+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:13:54.301052+0000","last_peered":"2024-09-14T04:13:54.301052+0000","last_clean":"2024-09-14T04:13:54.301052+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:13:54.301052+0000","last_undegraded":"2024-09-14T04:13:54.301052+0000","last_fullsized":"2024-09-14T04:13:54.301052+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":379,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.306488+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:13:54.306488+0000","last_peered":"2024-09-14T04:13:54.306488+0000","last_clean":"2024-09-14T04:13:54.306488+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:13:54.306488+0000","last_undegraded":"2024-09-14T04:13:54.306488+0000","last_fullsized":"2024-09-14T04:13:54.306488+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":328,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.305205+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:13:54.305205+0000","last_peered":"2024-09-14T04:13:54.305205+0000","last_clean":"2024-09-14T04:13:54.305205+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:13:54.305205+0000","last_undegraded":"2024-09-14T04:13:54.305205+0000","last_fullsized":"2024-09-14T04:13:54.305205+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":1,"reported_epoch":244,"state":"creating+peering","last_fresh":"2024-09-14T04:13:54.306108+0000","last_change":"2024-09-14T04:13:54.306108+0000","last_active":"2024-09-14T04:13:54.293151+0000","last_peered":"2024-09-14T04:13:54.293151+0000","last_clean":"2024-09-14T04:13:54.293151+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-14T04:13:54.306108+0000","last_undegraded":"2024-09-14T04:13:54.306108+0000","last_fullsized":"2024-09-14T04:13:54.306108+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":492,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310525+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:13:54.310525+0000","last_peered":"2024-09-14T04:13:54.310525+0000","last_clean":"2024-09-14T04:13:54.310525+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:13:54.310525+0000","last_undegraded":"2024-09-14T04:13:54.310525+0000","last_fullsized":"2024-09-14T04:13:54.310525+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":310,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.306391+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:13:54.306391+0000","last_peered":"2024-09-14T04:13:54.306391+0000","last_clean":"2024-09-14T04:13:54.306391+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:13:54.306391+0000","last_undegraded":"2024-09-14T04:13:54.306391+0000","last_fullsized":"2024-09-14T04:13:54.306391+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":261,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303656+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:13:54.303656+0000","last_peered":"2024-09-14T04:13:54.303656+0000","last_clean":"2024-09-14T04:13:54.303656+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:13:54.303656+0000","last_undegraded":"2024-09-14T04:13:54.303656+0000","last_fullsized":"2024-09-14T04:13:54.303656+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312418+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:13:56.312418+0000","last_peered":"2024-09-14T04:13:56.312418+0000","last_clean":"2024-09-14T04:13:56.312418+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:13:56.312418+0000","last_undegraded":"2024-09-14T04:13:56.312418+0000","last_fullsized":"2024-09-14T04:13:56.312418+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.315914+0000","last_change":"2024-09-14T04:13:55.315605+0000","last_active":"2024-09-14T04:13:55.315914+0000","last_peered":"2024-09-14T04:13:55.315914+0000","last_clean":"2024-09-14T04:13:55.315914+0000","last_became_active":"2024-09-14T04:13:55.315334+0000","last_became_peered":"2024-09-14T04:13:55.315334+0000","last_unstale":"2024-09-14T04:13:55.315914+0000","last_undegraded":"2024-09-14T04:13:55.315914+0000","last_fullsized":"2024-09-14T04:13:55.315914+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:15.039826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":450,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302504+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:13:54.302504+0000","last_peered":"2024-09-14T04:13:54.302504+0000","last_clean":"2024-09-14T04:13:54.302504+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:13:54.302504+0000","last_undegraded":"2024-09-14T04:13:54.302504+0000","last_fullsized":"2024-09-14T04:13:54.302504+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":179,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303878+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:13:54.303878+0000","last_peered":"2024-09-14T04:13:54.303878+0000","last_clean":"2024-09-14T04:13:54.303878+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:13:54.303878+0000","last_undegraded":"2024-09-14T04:13:54.303878+0000","last_fullsized":"2024-09-14T04:13:54.303878+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":311,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.304528+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:13:54.304528+0000","last_peered":"2024-09-14T04:13:54.304528+0000","last_clean":"2024-09-14T04:13:54.304528+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:13:54.304528+0000","last_undegraded":"2024-09-14T04:13:54.304528+0000","last_fullsized":"2024-09-14T04:13:54.304528+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":194,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302968+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:13:54.302968+0000","last_peered":"2024-09-14T04:13:54.302968+0000","last_clean":"2024-09-14T04:13:54.302968+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:13:54.302968+0000","last_undegraded":"2024-09-14T04:13:54.302968+0000","last_fullsized":"2024-09-14T04:13:54.302968+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":10,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.313643+0000","last_change":"2024-09-14T04:13:55.312894+0000","last_active":"2024-09-14T04:13:55.313643+0000","last_peered":"2024-09-14T04:13:55.313643+0000","last_clean":"2024-09-14T04:13:55.313643+0000","last_became_active":"2024-09-14T04:13:55.312568+0000","last_became_peered":"2024-09-14T04:13:55.312568+0000","last_unstale":"2024-09-14T04:13:55.313643+0000","last_undegraded":"2024-09-14T04:13:55.313643+0000","last_fullsized":"2024-09-14T04:13:55.313643+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:30.428884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":538,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.307749+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:13:55.307749+0000","last_peered":"2024-09-14T04:13:55.307749+0000","last_clean":"2024-09-14T04:13:55.307749+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:13:55.307749+0000","last_undegraded":"2024-09-14T04:13:55.307749+0000","last_fullsized":"2024-09-14T04:13:55.307749+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":89,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312375+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:13:56.312375+0000","last_peered":"2024-09-14T04:13:56.312375+0000","last_clean":"2024-09-14T04:13:56.312375+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:13:56.312375+0000","last_undegraded":"2024-09-14T04:13:56.312375+0000","last_fullsized":"2024-09-14T04:13:56.312375+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":244,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310697+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:13:54.310697+0000","last_peered":"2024-09-14T04:13:54.310697+0000","last_clean":"2024-09-14T04:13:54.310697+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:13:54.310697+0000","last_undegraded":"2024-09-14T04:13:54.310697+0000","last_fullsized":"2024-09-14T04:13:54.310697+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":200,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301963+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:13:54.301963+0000","last_peered":"2024-09-14T04:13:54.301963+0000","last_clean":"2024-09-14T04:13:54.301963+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:13:54.301963+0000","last_undegraded":"2024-09-14T04:13:54.301963+0000","last_fullsized":"2024-09-14T04:13:54.301963+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302237+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:13:54.302237+0000","last_peered":"2024-09-14T04:13:54.302237+0000","last_clean":"2024-09-14T04:13:54.302237+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:13:54.302237+0000","last_undegraded":"2024-09-14T04:13:54.302237+0000","last_fullsized":"2024-09-14T04:13:54.302237+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302470+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:13:54.302470+0000","last_peered":"2024-09-14T04:13:54.302470+0000","last_clean":"2024-09-14T04:13:54.302470+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:13:54.302470+0000","last_undegraded":"2024-09-14T04:13:54.302470+0000","last_fullsized":"2024-09-14T04:13:54.302470+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":310,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.305061+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:13:54.305061+0000","last_peered":"2024-09-14T04:13:54.305061+0000","last_clean":"2024-09-14T04:13:54.305061+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:13:54.305061+0000","last_undegraded":"2024-09-14T04:13:54.305061+0000","last_fullsized":"2024-09-14T04:13:54.305061+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":252,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303579+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:13:54.303579+0000","last_peered":"2024-09-14T04:13:54.303579+0000","last_clean":"2024-09-14T04:13:54.303579+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:13:54.303579+0000","last_undegraded":"2024-09-14T04:13:54.303579+0000","last_fullsized":"2024-09-14T04:13:54.303579+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300997+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:13:54.300997+0000","last_peered":"2024-09-14T04:13:54.300997+0000","last_clean":"2024-09-14T04:13:54.300997+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:13:54.300997+0000","last_undegraded":"2024-09-14T04:13:54.300997+0000","last_fullsized":"2024-09-14T04:13:54.300997+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":176,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.303198+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:13:54.303198+0000","last_peered":"2024-09-14T04:13:54.303198+0000","last_clean":"2024-09-14T04:13:54.303198+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:13:54.303198+0000","last_undegraded":"2024-09-14T04:13:54.303198+0000","last_fullsized":"2024-09-14T04:13:54.303198+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.311962+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:13:56.311962+0000","last_peered":"2024-09-14T04:13:56.311962+0000","last_clean":"2024-09-14T04:13:56.311962+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:13:56.311962+0000","last_undegraded":"2024-09-14T04:13:56.311962+0000","last_fullsized":"2024-09-14T04:13:56.311962+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-09-14T04:13:55.308657+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:13:55.308657+0000","last_peered":"2024-09-14T04:13:55.308657+0000","last_clean":"2024-09-14T04:13:55.308657+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:13:55.308657+0000","last_undegraded":"2024-09-14T04:13:55.308657+0000","last_fullsized":"2024-09-14T04:13:55.308657+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310621+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:13:54.310621+0000","last_peered":"2024-09-14T04:13:54.310621+0000","last_clean":"2024-09-14T04:13:54.310621+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:13:54.310621+0000","last_undegraded":"2024-09-14T04:13:54.310621+0000","last_fullsized":"2024-09-14T04:13:54.310621+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.311593+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:13:56.311593+0000","last_peered":"2024-09-14T04:13:56.311593+0000","last_clean":"2024-09-14T04:13:56.311593+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:13:56.311593+0000","last_undegraded":"2024-09-14T04:13:56.311593+0000","last_fullsized":"2024-09-14T04:13:56.311593+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312292+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:13:56.312292+0000","last_peered":"2024-09-14T04:13:56.312292+0000","last_clean":"2024-09-14T04:13:56.312292+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:13:56.312292+0000","last_undegraded":"2024-09-14T04:13:56.312292+0000","last_fullsized":"2024-09-14T04:13:56.312292+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":337,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.304085+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:13:54.304085+0000","last_peered":"2024-09-14T04:13:54.304085+0000","last_clean":"2024-09-14T04:13:54.304085+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:13:54.304085+0000","last_undegraded":"2024-09-14T04:13:54.304085+0000","last_fullsized":"2024-09-14T04:13:54.304085+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300892+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:13:54.300892+0000","last_peered":"2024-09-14T04:13:54.300892+0000","last_clean":"2024-09-14T04:13:54.300892+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:13:54.300892+0000","last_undegraded":"2024-09-14T04:13:54.300892+0000","last_fullsized":"2024-09-14T04:13:54.300892+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.311989+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:13:56.311989+0000","last_peered":"2024-09-14T04:13:56.311989+0000","last_clean":"2024-09-14T04:13:56.311989+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:13:56.311989+0000","last_undegraded":"2024-09-14T04:13:56.311989+0000","last_fullsized":"2024-09-14T04:13:56.311989+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":357,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310737+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:13:56.310737+0000","last_peered":"2024-09-14T04:13:56.310737+0000","last_clean":"2024-09-14T04:13:56.310737+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:13:56.310737+0000","last_undegraded":"2024-09-14T04:13:56.310737+0000","last_fullsized":"2024-09-14T04:13:56.310737+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312635+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:13:56.312635+0000","last_peered":"2024-09-14T04:13:56.312635+0000","last_clean":"2024-09-14T04:13:56.312635+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:13:56.312635+0000","last_undegraded":"2024-09-14T04:13:56.312635+0000","last_fullsized":"2024-09-14T04:13:56.312635+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301901+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:13:54.301901+0000","last_peered":"2024-09-14T04:13:54.301901+0000","last_clean":"2024-09-14T04:13:54.301901+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:13:54.301901+0000","last_undegraded":"2024-09-14T04:13:54.301901+0000","last_fullsized":"2024-09-14T04:13:54.301901+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":252,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.300927+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:13:54.300927+0000","last_peered":"2024-09-14T04:13:54.300927+0000","last_clean":"2024-09-14T04:13:54.300927+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:13:54.300927+0000","last_undegraded":"2024-09-14T04:13:54.300927+0000","last_fullsized":"2024-09-14T04:13:54.300927+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.302035+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:13:54.302035+0000","last_peered":"2024-09-14T04:13:54.302035+0000","last_clean":"2024-09-14T04:13:54.302035+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:13:54.302035+0000","last_undegraded":"2024-09-14T04:13:54.302035+0000","last_fullsized":"2024-09-14T04:13:54.302035+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":239,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301008+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:13:54.301008+0000","last_peered":"2024-09-14T04:13:54.301008+0000","last_clean":"2024-09-14T04:13:54.301008+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:13:54.301008+0000","last_undegraded":"2024-09-14T04:13:54.301008+0000","last_fullsized":"2024-09-14T04:13:54.301008+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":319,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.310410+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:13:56.310410+0000","last_peered":"2024-09-14T04:13:56.310410+0000","last_clean":"2024-09-14T04:13:56.310410+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:13:56.310410+0000","last_undegraded":"2024-09-14T04:13:56.310410+0000","last_fullsized":"2024-09-14T04:13:56.310410+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":121,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.310624+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:13:54.310624+0000","last_peered":"2024-09-14T04:13:54.310624+0000","last_clean":"2024-09-14T04:13:54.310624+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:13:54.310624+0000","last_undegraded":"2024-09-14T04:13:54.310624+0000","last_fullsized":"2024-09-14T04:13:54.310624+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":212,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-09-14T04:13:54.301802+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:13:54.301802+0000","last_peered":"2024-09-14T04:13:54.301802+0000","last_clean":"2024-09-14T04:13:54.301802+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:13:54.301802+0000","last_undegraded":"2024-09-14T04:13:54.301802+0000","last_fullsized":"2024-09-14T04:13:54.301802+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":91,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-09-14T04:13:56.312088+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:13:56.312088+0000","last_peered":"2024-09-14T04:13:56.312088+0000","last_clean":"2024-09-14T04:13:56.312088+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:13:56.312088+0000","last_undegraded":"2024-09-14T04:13:56.312088+0000","last_fullsized":"2024-09-14T04:13:56.312088+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201680,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":800952,"kb_used_data":3628,"kb_used_omap":1,"kb_used_meta":797310,"kb_avail":92944200,"statfs":{"total":95995035648,"available":95174860800,"internally_reserved":0,"allocated":3715072,"data_stored":2152192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":816445899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463318,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661996,"kb_used_data":3296,"kb_used_omap":1,"kb_used_meta":658686,"kb_avail":93083156,"statfs":{"total":95995035648,"available":95317151744,"internally_reserved":0,"allocated":3375104,"data_stored":1823865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":674494919},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724954,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027628,"kb_used_data":3872,"kb_used_omap":1,"kb_used_meta":1023742,"kb_avail":92717524,"statfs":{"total":95995035648,"available":94942744576,"internally_reserved":0,"allocated":3964928,"data_stored":2410332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048312262},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986590,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850676,"kb_used_data":3304,"kb_used_omap":1,"kb_used_meta":847358,"kb_avail":92894476,"statfs":{"total":95995035648,"available":95123943424,"internally_reserved":0,"allocated":3383296,"data_stored":1824203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":867695046},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748619,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412136,"kb_used_data":3868,"kb_used_omap":4,"kb_used_meta":408251,"kb_avail":93333016,"statfs":{"total":95995035648,"available":95573008384,"internally_reserved":0,"allocated":3960832,"data_stored":2410044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":418049580},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912146,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":573688,"kb_used_data":288236,"kb_used_omap":6,"kb_used_meta":285433,"kb_avail":93171464,"statfs":{"total":95995035648,"available":95407579136,"internally_reserved":0,"allocated":295153664,"data_stored":293595766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":292283961},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468162,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":512788,"kb_used_data":3272,"kb_used_omap":5,"kb_used_meta":509498,"kb_avail":93232364,"statfs":{"total":95995035648,"available":95469940736,"internally_reserved":0,"allocated":3350528,"data_stored":1809700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":521726303},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342278,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":223424,"kb_used_data":3308,"kb_used_omap":10,"kb_used_meta":220085,"kb_avail":93521728,"statfs":{"total":95995035648,"available":95766249472,"internally_reserved":0,"allocated":3387392,"data_stored":1828972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":225367250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:14:01.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:01.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:01.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:01.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:02.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:02.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:02.136 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:02.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:02.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:02.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:02.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:02.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:02 smithi001 ceph-mon[107399]: pgmap v860: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:02 smithi001 ceph-mon[112165]: pgmap v860: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:02.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:02.521 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:02 smithi012 ceph-mon[101180]: pgmap v860: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:02.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:02.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:02.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:02.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:02.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:02.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:02.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:02.917 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:03.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:03.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:03.128 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:03.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:03.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:03.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:03.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:03.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:03.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:03.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:03.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:03.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:03.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:03.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:03.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:03.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:03.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:03.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:03.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:04.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:04.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:04.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:04.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:04.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:04.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:04.331 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:14:04.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:04.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:04.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:04.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:04 smithi012 ceph-mon[101180]: pgmap v861: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:04 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:14:04.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:04.672 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:04 smithi001 ceph-mon[107399]: pgmap v861: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:04 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:14:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:04 smithi001 ceph-mon[112165]: pgmap v861: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:04 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:14:04.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:04.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:04.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:14:04.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:04.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:04.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:05.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:05.051 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:05.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:05.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:05.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:05.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:05.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:05.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:05.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:05 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:14:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:05 smithi012 ceph-mon[101180]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T04:14:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:05.645 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[107399]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3890876545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[112165]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:05.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:05.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:05.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:05.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:05.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:05.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:05.949 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:05.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:06.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:06.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:06.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:06.201 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:14:06.201 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:14:06.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:06.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:06.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:06.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:06.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:06.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:06.588 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:06.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:06 smithi012 ceph-mon[101180]: pgmap v863: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:06.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:06.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:06 smithi001 ceph-mon[107399]: pgmap v863: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:06.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:06 smithi001 ceph-mon[112165]: pgmap v863: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:06.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:06.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:06.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:06.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:06] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-14T04:14:06.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:14:06.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:06.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:07.071 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:07.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:07.152 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:14:07.152 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:14:07.173 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:14:07.173 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:14:07.193 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:14:07.193 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:14:07.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:07.211 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:14:07.211 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:14:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:07.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:07.227 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:14:07.228 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:14:07.244 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:14:07.244 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:14:07.261 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:14:07.262 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:14:07.278 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:14:07.278 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:14:07.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:07.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:07.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:07.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:07.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2726689507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2726689507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:07.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:07.590 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:07.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2726689507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:07.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:14:07.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:07.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:07.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:07.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:07.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:07.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:07.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:07.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:08.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:08.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:08.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:08.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:08.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:08.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:08.342 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-09-14T04:14:08.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.464 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:08.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:08.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[107399]: pgmap v864: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/337686812' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[112165]: pgmap v864: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/337686812' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:08 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:08.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:08 smithi012 ceph-mon[101180]: pgmap v864: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/337686812' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:08 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:08.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:08.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:08.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:08.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:08.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:08.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:08.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:09.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:09.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:09.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:09.216 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:09.238 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:09.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:09.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:09.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:09.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:09.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:09.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:09 smithi012 ceph-mon[101180]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T04:14:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:09.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:09.675 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:09.675 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:09 smithi001 ceph-mon[107399]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T04:14:09.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:09.676 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:09 smithi001 ceph-mon[112165]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T04:14:09.698 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:09.713 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:09.720 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:09.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:09.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:09.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:09.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:09.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:09.977 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:09.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:09.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:10.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:10.103 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:10.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:10.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:10.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:10.353 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:10.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:10.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:10.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:14:10.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:10 smithi012 ceph-mon[101180]: pgmap v866: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:10 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:14:10.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:10 smithi001 ceph-mon[112165]: pgmap v866: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:10.599 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:10 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:14:10.599 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:10 smithi001 ceph-mon[107399]: pgmap v866: 99 pgs: 99 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:14:10.599 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:10 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:14:10.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:10.698 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-09-14T04:14:10.701 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:14:10.702 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:14:10.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:10.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:10.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:10.763 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:14:10.763 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:14:10.763 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_228075 2024-09-14T04:14:10.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:14:10.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:14:10.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:10.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:10.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:10.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:10.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:11.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:11.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:11.155 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:11.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:11.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:11.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:11.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:11.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:11.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:11.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:11.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:11.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:11.763 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3810 3794 237.119 237.125 0.00440698 0.00420316 2024-09-14T04:14:11.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:11.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:11.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:11.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:11.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:12.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:12.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:12.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:12.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:12.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:12.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:12.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:12.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:12.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:12.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:12 smithi001 ceph-mon[112165]: pgmap v867: 99 pgs: 99 active+clean; 18 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-14T04:14:12.677 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:12 smithi001 ceph-mon[107399]: pgmap v867: 99 pgs: 99 active+clean; 18 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-14T04:14:12.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:12.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:12.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 7279 7263 226.956 216.812 0.00311403 0.00439494 2024-09-14T04:14:12.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:12.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:12 smithi012 ceph-mon[101180]: pgmap v867: 99 pgs: 99 active+clean; 18 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-14T04:14:12.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:12.973 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:12.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:13.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:13.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:13.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:13.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:13.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:13.402 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:13.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:13.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:13.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:13.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:13.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 10099 10083 210.048 176.25 0.00388184 0.00475387 2024-09-14T04:14:13.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:13.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:13.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:13.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:13.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:14.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:14.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:14.186 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:14.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:14.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:14.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:14.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:14.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:14.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:14.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:14.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:14 smithi001 ceph-mon[107399]: pgmap v868: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s wr, 298 op/s 2024-09-14T04:14:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:14 smithi001 ceph-mon[112165]: pgmap v868: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s wr, 298 op/s 2024-09-14T04:14:14.741 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:14.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 12993 12977 202.751 180.875 0.0041253 0.00492482 2024-09-14T04:14:14.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:14 smithi012 ceph-mon[101180]: pgmap v868: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s wr, 298 op/s 2024-09-14T04:14:14.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:14.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:14.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:14.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:15.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:15.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:15.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:15.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:15.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:15.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:15.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:15.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:15.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:15.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:15.641 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:15.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:15.764 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 15831 15815 197.672 177.375 0.00668402 0.00505182 2024-09-14T04:14:15.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:15.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:15.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:15.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:16.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:16.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:16.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:16.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:16.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:16.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.353 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:16.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:16.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:16.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:16.624 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:14:16.624 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:16 smithi001 ceph-mon[112165]: pgmap v869: 99 pgs: 99 active+clean; 313 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s 2024-09-14T04:14:16.625 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:16 smithi001 ceph-mon[107399]: pgmap v869: 99 pgs: 99 active+clean; 313 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s 2024-09-14T04:14:16.627 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:14:16.642 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:16.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:16.765 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 18427 18411 191.729 162.25 0.00739765 0.00520762 2024-09-14T04:14:16.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:16 smithi012 ceph-mon[101180]: pgmap v869: 99 pgs: 99 active+clean; 313 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s 2024-09-14T04:14:16.920 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:16] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-14T04:14:16.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:16.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:16.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:17.051 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:14:17.051 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:14:17.052 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:14:17.052 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:14:17.052 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:14:17.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:17.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:17.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:17.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:17.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:17.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:17.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:17.518 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:17 smithi012 ceph-mon[101180]: from='client.45371 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:17.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:17.689 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[107399]: from='client.45371 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:17.689 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.689 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[112165]: from='client.45371 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:14:17.690 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.690 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:17.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:17.765 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 21102 21086 188.218 167.188 0.00607944 0.00530671 2024-09-14T04:14:17.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:17.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:17.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:17.939 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:17.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:18.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:18.073 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":869,"stamp":"2024-09-14T04:14:15.218565+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":328210405,"num_objects":5025,"num_object_clones":0,"num_object_copies":15075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5063,"num_write_kb":319969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":918994,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6697,"ondisk_log_size":6697,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5942828,"kb_used_data":1134180,"kb_used_omap":33,"kb_used_meta":4808542,"kb_avail":744018388,"statfs":{"total":767960285184,"available":761874829312,"internally_reserved":0,"allocated":1161400320,"data_stored":1148880487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":4923947444},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4999,"num_write_kb":319936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002694"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":286,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.670776+0000","last_change":"2024-09-14T04:07:20.033222+0000","last_active":"2024-09-14T04:14:08.670776+0000","last_peered":"2024-09-14T04:14:08.670776+0000","last_clean":"2024-09-14T04:14:08.670776+0000","last_became_active":"2024-09-14T04:07:20.032883+0000","last_became_peered":"2024-09-14T04:07:20.032883+0000","last_unstale":"2024-09-14T04:14:08.670776+0000","last_undegraded":"2024-09-14T04:14:08.670776+0000","last_fullsized":"2024-09-14T04:14:08.670776+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:03.517388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":239,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341720+0000","last_change":"2024-09-14T04:08:15.318275+0000","last_active":"2024-09-14T04:14:08.341720+0000","last_peered":"2024-09-14T04:14:08.341720+0000","last_clean":"2024-09-14T04:14:08.341720+0000","last_became_active":"2024-09-14T04:08:15.317205+0000","last_became_peered":"2024-09-14T04:08:15.317205+0000","last_unstale":"2024-09-14T04:14:08.341720+0000","last_undegraded":"2024-09-14T04:14:08.341720+0000","last_fullsized":"2024-09-14T04:14:08.341720+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:04.494576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332819+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:14:04.332819+0000","last_peered":"2024-09-14T04:14:04.332819+0000","last_clean":"2024-09-14T04:14:04.332819+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:14:04.332819+0000","last_undegraded":"2024-09-14T04:14:04.332819+0000","last_fullsized":"2024-09-14T04:14:04.332819+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341185+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:14:08.341185+0000","last_peered":"2024-09-14T04:14:08.341185+0000","last_clean":"2024-09-14T04:14:08.341185+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:14:08.341185+0000","last_undegraded":"2024-09-14T04:14:08.341185+0000","last_fullsized":"2024-09-14T04:14:08.341185+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341694+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:14:08.341694+0000","last_peered":"2024-09-14T04:14:08.341694+0000","last_clean":"2024-09-14T04:14:08.341694+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:14:08.341694+0000","last_undegraded":"2024-09-14T04:14:08.341694+0000","last_fullsized":"2024-09-14T04:14:08.341694+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368898+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:14:04.368898+0000","last_peered":"2024-09-14T04:14:04.368898+0000","last_clean":"2024-09-14T04:14:04.368898+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:14:04.368898+0000","last_undegraded":"2024-09-14T04:14:04.368898+0000","last_fullsized":"2024-09-14T04:14:04.368898+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":197,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.070177+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:14:09.070177+0000","last_peered":"2024-09-14T04:14:09.070177+0000","last_clean":"2024-09-14T04:14:09.070177+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:14:09.070177+0000","last_undegraded":"2024-09-14T04:14:09.070177+0000","last_fullsized":"2024-09-14T04:14:09.070177+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":299,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332610+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:14:04.332610+0000","last_peered":"2024-09-14T04:14:04.332610+0000","last_clean":"2024-09-14T04:14:04.332610+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:14:04.332610+0000","last_undegraded":"2024-09-14T04:14:04.332610+0000","last_fullsized":"2024-09-14T04:14:04.332610+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"248'294","reported_seq":307,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.946565+0000","last_change":"2024-09-14T04:13:55.311089+0000","last_active":"2024-09-14T04:14:11.946565+0000","last_peered":"2024-09-14T04:14:11.946565+0000","last_clean":"2024-09-14T04:14:11.946565+0000","last_became_active":"2024-09-14T04:13:55.310788+0000","last_became_peered":"2024-09-14T04:13:55.310788+0000","last_unstale":"2024-09-14T04:14:11.946565+0000","last_undegraded":"2024-09-14T04:14:11.946565+0000","last_fullsized":"2024-09-14T04:14:11.946565+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":294,"log_dups_size":0,"ondisk_log_size":294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:55.525731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267584,"num_objects":294,"num_object_clones":0,"num_object_copies":882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":294,"num_write_kb":18816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":259,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.369540+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:14:04.369540+0000","last_peered":"2024-09-14T04:14:04.369540+0000","last_clean":"2024-09-14T04:14:04.369540+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:14:04.369540+0000","last_undegraded":"2024-09-14T04:14:04.369540+0000","last_fullsized":"2024-09-14T04:14:04.369540+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":59,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.068507+0000","last_change":"2024-09-14T04:09:50.223132+0000","last_active":"2024-09-14T04:14:09.068507+0000","last_peered":"2024-09-14T04:14:09.068507+0000","last_clean":"2024-09-14T04:14:09.068507+0000","last_became_active":"2024-09-14T04:09:50.222715+0000","last_became_peered":"2024-09-14T04:09:50.222715+0000","last_unstale":"2024-09-14T04:14:09.068507+0000","last_undegraded":"2024-09-14T04:14:09.068507+0000","last_fullsized":"2024-09-14T04:14:09.068507+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:04:36.626779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"248'668","reported_seq":687,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:13.907659+0000","last_change":"2024-09-14T04:13:55.317315+0000","last_active":"2024-09-14T04:14:13.907659+0000","last_peered":"2024-09-14T04:14:13.907659+0000","last_clean":"2024-09-14T04:14:13.907659+0000","last_became_active":"2024-09-14T04:13:55.316972+0000","last_became_peered":"2024-09-14T04:13:55.316972+0000","last_unstale":"2024-09-14T04:14:13.907659+0000","last_undegraded":"2024-09-14T04:14:13.907659+0000","last_fullsized":"2024-09-14T04:14:13.907659+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":668,"log_dups_size":0,"ondisk_log_size":668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:49.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43778048,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":668,"num_write_kb":42752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":259,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.340925+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:14:08.340925+0000","last_peered":"2024-09-14T04:14:08.340925+0000","last_clean":"2024-09-14T04:14:08.340925+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:14:08.340925+0000","last_undegraded":"2024-09-14T04:14:08.340925+0000","last_fullsized":"2024-09-14T04:14:08.340925+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":250,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368299+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:14:04.368299+0000","last_peered":"2024-09-14T04:14:04.368299+0000","last_clean":"2024-09-14T04:14:04.368299+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:14:04.368299+0000","last_undegraded":"2024-09-14T04:14:04.368299+0000","last_fullsized":"2024-09-14T04:14:04.368299+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"248'90","reported_seq":102,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.168932+0000","last_change":"2024-09-14T04:13:55.316081+0000","last_active":"2024-09-14T04:14:11.168932+0000","last_peered":"2024-09-14T04:14:11.168932+0000","last_clean":"2024-09-14T04:14:11.168932+0000","last_became_active":"2024-09-14T04:13:55.315775+0000","last_became_peered":"2024-09-14T04:13:55.315775+0000","last_unstale":"2024-09-14T04:14:11.168932+0000","last_undegraded":"2024-09-14T04:14:11.168932+0000","last_fullsized":"2024-09-14T04:14:11.168932+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:39:05.829936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5898240,"num_objects":90,"num_object_clones":0,"num_object_copies":270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90,"num_write_kb":5760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":230,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341246+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:14:08.341246+0000","last_peered":"2024-09-14T04:14:08.341246+0000","last_clean":"2024-09-14T04:14:08.341246+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:14:08.341246+0000","last_undegraded":"2024-09-14T04:14:08.341246+0000","last_fullsized":"2024-09-14T04:14:08.341246+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333061+0000","last_change":"2024-09-14T03:57:55.647540+0000","last_active":"2024-09-14T04:14:04.333061+0000","last_peered":"2024-09-14T04:14:04.333061+0000","last_clean":"2024-09-14T04:14:04.333061+0000","last_became_active":"2024-09-14T03:57:55.647050+0000","last_became_peered":"2024-09-14T03:57:55.647050+0000","last_unstale":"2024-09-14T04:14:04.333061+0000","last_undegraded":"2024-09-14T04:14:04.333061+0000","last_fullsized":"2024-09-14T04:14:04.333061+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:38.562741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"248'87","reported_seq":100,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.168977+0000","last_change":"2024-09-14T04:13:55.315421+0000","last_active":"2024-09-14T04:14:11.168977+0000","last_peered":"2024-09-14T04:14:11.168977+0000","last_clean":"2024-09-14T04:14:11.168977+0000","last_became_active":"2024-09-14T04:13:55.315144+0000","last_became_peered":"2024-09-14T04:13:55.315144+0000","last_unstale":"2024-09-14T04:14:11.168977+0000","last_undegraded":"2024-09-14T04:14:11.168977+0000","last_fullsized":"2024-09-14T04:14:11.168977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:17.194435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87,"num_write_kb":5568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:06.953363+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:14:06.953363+0000","last_peered":"2024-09-14T04:14:06.953363+0000","last_clean":"2024-09-14T04:14:06.953363+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:14:06.953363+0000","last_undegraded":"2024-09-14T04:14:06.953363+0000","last_fullsized":"2024-09-14T04:14:06.953363+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341812+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:14:08.341812+0000","last_peered":"2024-09-14T04:14:08.341812+0000","last_clean":"2024-09-14T04:14:08.341812+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:14:08.341812+0000","last_undegraded":"2024-09-14T04:14:08.341812+0000","last_fullsized":"2024-09-14T04:14:08.341812+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"248'157","reported_seq":170,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.432873+0000","last_change":"2024-09-14T04:13:55.315439+0000","last_active":"2024-09-14T04:14:11.432873+0000","last_peered":"2024-09-14T04:14:11.432873+0000","last_clean":"2024-09-14T04:14:11.432873+0000","last_became_active":"2024-09-14T04:13:55.315149+0000","last_became_peered":"2024-09-14T04:13:55.315149+0000","last_unstale":"2024-09-14T04:14:11.432873+0000","last_undegraded":"2024-09-14T04:14:11.432873+0000","last_fullsized":"2024-09-14T04:14:11.432873+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:40.970593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10289152,"num_objects":157,"num_object_clones":0,"num_object_copies":471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":157,"num_write_kb":10048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":600,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368660+0000","last_change":"2024-09-14T04:08:32.948759+0000","last_active":"2024-09-14T04:14:04.368660+0000","last_peered":"2024-09-14T04:14:04.368660+0000","last_clean":"2024-09-14T04:14:04.368660+0000","last_became_active":"2024-09-14T04:08:31.385422+0000","last_became_peered":"2024-09-14T04:08:31.385422+0000","last_unstale":"2024-09-14T04:14:04.368660+0000","last_undegraded":"2024-09-14T04:14:04.368660+0000","last_fullsized":"2024-09-14T04:14:04.368660+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:22.564562+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":93,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341474+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:14:08.341474+0000","last_peered":"2024-09-14T04:14:08.341474+0000","last_clean":"2024-09-14T04:14:08.341474+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:14:08.341474+0000","last_undegraded":"2024-09-14T04:14:08.341474+0000","last_fullsized":"2024-09-14T04:14:08.341474+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:05.891465+0000","last_change":"2024-09-14T04:06:46.335857+0000","last_active":"2024-09-14T04:14:05.891465+0000","last_peered":"2024-09-14T04:14:05.891465+0000","last_clean":"2024-09-14T04:14:05.891465+0000","last_became_active":"2024-09-14T04:06:46.335266+0000","last_became_peered":"2024-09-14T04:06:46.335266+0000","last_unstale":"2024-09-14T04:14:05.891465+0000","last_undegraded":"2024-09-14T04:14:05.891465+0000","last_fullsized":"2024-09-14T04:14:05.891465+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:08.595035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"248'463","reported_seq":482,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:12.772576+0000","last_change":"2024-09-14T04:13:55.315267+0000","last_active":"2024-09-14T04:14:12.772576+0000","last_peered":"2024-09-14T04:14:12.772576+0000","last_clean":"2024-09-14T04:14:12.772576+0000","last_became_active":"2024-09-14T04:13:55.314973+0000","last_became_peered":"2024-09-14T04:13:55.314973+0000","last_unstale":"2024-09-14T04:14:12.772576+0000","last_undegraded":"2024-09-14T04:14:12.772576+0000","last_fullsized":"2024-09-14T04:14:12.772576+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":463,"log_dups_size":0,"ondisk_log_size":463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:50:08.424954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30343168,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":463,"num_write_kb":29632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":469,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332058+0000","last_change":"2024-09-14T04:09:53.254707+0000","last_active":"2024-09-14T04:14:04.332058+0000","last_peered":"2024-09-14T04:14:04.332058+0000","last_clean":"2024-09-14T04:14:04.332058+0000","last_became_active":"2024-09-14T04:09:53.254228+0000","last_became_peered":"2024-09-14T04:09:53.254228+0000","last_unstale":"2024-09-14T04:14:04.332058+0000","last_undegraded":"2024-09-14T04:14:04.332058+0000","last_fullsized":"2024-09-14T04:14:04.332058+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:43:20.411833+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333209+0000","last_change":"2024-09-14T04:08:15.317521+0000","last_active":"2024-09-14T04:14:04.333209+0000","last_peered":"2024-09-14T04:14:04.333209+0000","last_clean":"2024-09-14T04:14:04.333209+0000","last_became_active":"2024-09-14T04:08:15.317214+0000","last_became_peered":"2024-09-14T04:08:15.317214+0000","last_unstale":"2024-09-14T04:14:04.333209+0000","last_undegraded":"2024-09-14T04:14:04.333209+0000","last_fullsized":"2024-09-14T04:14:04.333209+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:40.709544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":39,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.369449+0000","last_change":"2024-09-14T04:11:13.250259+0000","last_active":"2024-09-14T04:14:04.369449+0000","last_peered":"2024-09-14T04:14:04.369449+0000","last_clean":"2024-09-14T04:14:04.369449+0000","last_became_active":"2024-09-14T04:11:13.249939+0000","last_became_peered":"2024-09-14T04:11:13.249939+0000","last_unstale":"2024-09-14T04:14:04.369449+0000","last_undegraded":"2024-09-14T04:14:04.369449+0000","last_fullsized":"2024-09-14T04:14:04.369449+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:55.797246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"248'149","reported_seq":162,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.436635+0000","last_change":"2024-09-14T04:13:55.315758+0000","last_active":"2024-09-14T04:14:11.436635+0000","last_peered":"2024-09-14T04:14:11.436635+0000","last_clean":"2024-09-14T04:14:11.436635+0000","last_became_active":"2024-09-14T04:13:55.314723+0000","last_became_peered":"2024-09-14T04:13:55.314723+0000","last_unstale":"2024-09-14T04:14:11.436635+0000","last_undegraded":"2024-09-14T04:14:11.436635+0000","last_fullsized":"2024-09-14T04:14:11.436635+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:53.745702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9764864,"num_objects":149,"num_object_clones":0,"num_object_copies":447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":149,"num_write_kb":9536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":39,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368150+0000","last_change":"2024-09-14T04:11:13.253425+0000","last_active":"2024-09-14T04:14:04.368150+0000","last_peered":"2024-09-14T04:14:04.368150+0000","last_clean":"2024-09-14T04:14:04.368150+0000","last_became_active":"2024-09-14T04:11:13.253041+0000","last_became_peered":"2024-09-14T04:11:13.253041+0000","last_unstale":"2024-09-14T04:14:04.368150+0000","last_undegraded":"2024-09-14T04:14:04.368150+0000","last_fullsized":"2024-09-14T04:14:04.368150+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:31.245172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":217,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069124+0000","last_change":"2024-09-14T03:57:55.658117+0000","last_active":"2024-09-14T04:14:09.069124+0000","last_peered":"2024-09-14T04:14:09.069124+0000","last_clean":"2024-09-14T04:14:09.069124+0000","last_became_active":"2024-09-14T03:57:55.657641+0000","last_became_peered":"2024-09-14T03:57:55.657641+0000","last_unstale":"2024-09-14T04:14:09.069124+0000","last_undegraded":"2024-09-14T04:14:09.069124+0000","last_fullsized":"2024-09-14T04:14:09.069124+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:46:59.610984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"248'866","reported_seq":885,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:14.990039+0000","last_change":"2024-09-14T04:13:55.312520+0000","last_active":"2024-09-14T04:14:14.990039+0000","last_peered":"2024-09-14T04:14:14.990039+0000","last_clean":"2024-09-14T04:14:14.990039+0000","last_became_active":"2024-09-14T04:13:55.312226+0000","last_became_peered":"2024-09-14T04:13:55.312226+0000","last_unstale":"2024-09-14T04:14:14.990039+0000","last_undegraded":"2024-09-14T04:14:14.990039+0000","last_fullsized":"2024-09-14T04:14:14.990039+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":866,"log_dups_size":0,"ondisk_log_size":866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:23:46.283362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":866,"num_write_kb":55424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":453,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332305+0000","last_change":"2024-09-14T04:09:10.340601+0000","last_active":"2024-09-14T04:14:04.332305+0000","last_peered":"2024-09-14T04:14:04.332305+0000","last_clean":"2024-09-14T04:14:04.332305+0000","last_became_active":"2024-09-14T04:08:31.387397+0000","last_became_peered":"2024-09-14T04:08:31.387397+0000","last_unstale":"2024-09-14T04:14:04.332305+0000","last_undegraded":"2024-09-14T04:14:04.332305+0000","last_fullsized":"2024-09-14T04:14:04.332305+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:06.412468+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":197,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.068397+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:14:09.068397+0000","last_peered":"2024-09-14T04:14:09.068397+0000","last_clean":"2024-09-14T04:14:09.068397+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:14:09.068397+0000","last_undegraded":"2024-09-14T04:14:09.068397+0000","last_fullsized":"2024-09-14T04:14:09.068397+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":333,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.070033+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:14:09.070033+0000","last_peered":"2024-09-14T04:14:09.070033+0000","last_clean":"2024-09-14T04:14:09.070033+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:14:09.070033+0000","last_undegraded":"2024-09-14T04:14:09.070033+0000","last_fullsized":"2024-09-14T04:14:09.070033+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"248'147","reported_seq":160,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.420266+0000","last_change":"2024-09-14T04:13:55.314904+0000","last_active":"2024-09-14T04:14:11.420266+0000","last_peered":"2024-09-14T04:14:11.420266+0000","last_clean":"2024-09-14T04:14:11.420266+0000","last_became_active":"2024-09-14T04:13:55.314584+0000","last_became_peered":"2024-09-14T04:13:55.314584+0000","last_unstale":"2024-09-14T04:14:11.420266+0000","last_undegraded":"2024-09-14T04:14:11.420266+0000","last_fullsized":"2024-09-14T04:14:11.420266+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:15:01.720889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9633792,"num_objects":147,"num_object_clones":0,"num_object_copies":441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":147,"num_write_kb":9408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":618,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368509+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:14:04.368509+0000","last_peered":"2024-09-14T04:14:04.368509+0000","last_clean":"2024-09-14T04:14:04.368509+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:14:04.368509+0000","last_undegraded":"2024-09-14T04:14:04.368509+0000","last_fullsized":"2024-09-14T04:14:04.368509+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":232,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332780+0000","last_change":"2024-09-14T04:01:59.055329+0000","last_active":"2024-09-14T04:14:04.332780+0000","last_peered":"2024-09-14T04:14:04.332780+0000","last_clean":"2024-09-14T04:14:04.332780+0000","last_became_active":"2024-09-14T04:01:59.054995+0000","last_became_peered":"2024-09-14T04:01:59.054995+0000","last_unstale":"2024-09-14T04:14:04.332780+0000","last_undegraded":"2024-09-14T04:14:04.332780+0000","last_fullsized":"2024-09-14T04:14:04.332780+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:20.117506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":179,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332450+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:14:04.332450+0000","last_peered":"2024-09-14T04:14:04.332450+0000","last_clean":"2024-09-14T04:14:04.332450+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:14:04.332450+0000","last_undegraded":"2024-09-14T04:14:04.332450+0000","last_fullsized":"2024-09-14T04:14:04.332450+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":201,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332578+0000","last_change":"2024-09-14T03:42:35.662745+0000","last_active":"2024-09-14T04:14:04.332578+0000","last_peered":"2024-09-14T04:14:04.332578+0000","last_clean":"2024-09-14T04:14:04.332578+0000","last_became_active":"2024-09-14T03:42:35.662401+0000","last_became_peered":"2024-09-14T03:42:35.662401+0000","last_unstale":"2024-09-14T04:14:04.332578+0000","last_undegraded":"2024-09-14T04:14:04.332578+0000","last_fullsized":"2024-09-14T04:14:04.332578+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:38.206803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"248'238","reported_seq":257,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.828652+0000","last_change":"2024-09-14T04:13:55.313345+0000","last_active":"2024-09-14T04:14:11.828652+0000","last_peered":"2024-09-14T04:14:11.828652+0000","last_clean":"2024-09-14T04:14:11.828652+0000","last_became_active":"2024-09-14T04:13:55.313088+0000","last_became_peered":"2024-09-14T04:13:55.313088+0000","last_unstale":"2024-09-14T04:14:11.828652+0000","last_undegraded":"2024-09-14T04:14:11.828652+0000","last_fullsized":"2024-09-14T04:14:11.828652+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":238,"log_dups_size":0,"ondisk_log_size":238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:55:44.190601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":238,"num_write_kb":15232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":516,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069291+0000","last_change":"2024-09-14T04:08:32.190720+0000","last_active":"2024-09-14T04:14:09.069291+0000","last_peered":"2024-09-14T04:14:09.069291+0000","last_clean":"2024-09-14T04:14:09.069291+0000","last_became_active":"2024-09-14T04:08:31.385020+0000","last_became_peered":"2024-09-14T04:08:31.385020+0000","last_unstale":"2024-09-14T04:14:09.069291+0000","last_undegraded":"2024-09-14T04:14:09.069291+0000","last_fullsized":"2024-09-14T04:14:09.069291+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:29.472303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":262,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368411+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:14:04.368411+0000","last_peered":"2024-09-14T04:14:04.368411+0000","last_clean":"2024-09-14T04:14:04.368411+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:14:04.368411+0000","last_undegraded":"2024-09-14T04:14:04.368411+0000","last_fullsized":"2024-09-14T04:14:04.368411+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":312,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341129+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:14:08.341129+0000","last_peered":"2024-09-14T04:14:08.341129+0000","last_clean":"2024-09-14T04:14:08.341129+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:14:08.341129+0000","last_undegraded":"2024-09-14T04:14:08.341129+0000","last_fullsized":"2024-09-14T04:14:08.341129+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341699+0000","last_change":"2024-09-14T04:10:35.382986+0000","last_active":"2024-09-14T04:14:08.341699+0000","last_peered":"2024-09-14T04:14:08.341699+0000","last_clean":"2024-09-14T04:14:08.341699+0000","last_became_active":"2024-09-14T04:10:35.382409+0000","last_became_peered":"2024-09-14T04:10:35.382409+0000","last_unstale":"2024-09-14T04:14:08.341699+0000","last_undegraded":"2024-09-14T04:14:08.341699+0000","last_fullsized":"2024-09-14T04:14:08.341699+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:26.013912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"248'155","reported_seq":168,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.434309+0000","last_change":"2024-09-14T04:13:55.314961+0000","last_active":"2024-09-14T04:14:11.434309+0000","last_peered":"2024-09-14T04:14:11.434309+0000","last_clean":"2024-09-14T04:14:11.434309+0000","last_became_active":"2024-09-14T04:13:55.314601+0000","last_became_peered":"2024-09-14T04:13:55.314601+0000","last_unstale":"2024-09-14T04:14:11.434309+0000","last_undegraded":"2024-09-14T04:14:11.434309+0000","last_fullsized":"2024-09-14T04:14:11.434309+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:50:32.900254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10158080,"num_objects":155,"num_object_clones":0,"num_object_copies":465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":155,"num_write_kb":9920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":481,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:06.168937+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:14:06.168937+0000","last_peered":"2024-09-14T04:14:06.168937+0000","last_clean":"2024-09-14T04:14:06.168937+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:14:06.168937+0000","last_undegraded":"2024-09-14T04:14:06.168937+0000","last_fullsized":"2024-09-14T04:14:06.168937+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":375,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.670364+0000","last_change":"2024-09-14T04:08:16.317280+0000","last_active":"2024-09-14T04:14:08.670364+0000","last_peered":"2024-09-14T04:14:08.670364+0000","last_clean":"2024-09-14T04:14:08.670364+0000","last_became_active":"2024-09-14T04:08:16.316880+0000","last_became_peered":"2024-09-14T04:08:16.316880+0000","last_unstale":"2024-09-14T04:14:08.670364+0000","last_undegraded":"2024-09-14T04:14:08.670364+0000","last_fullsized":"2024-09-14T04:14:08.670364+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:43.771695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":319,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069087+0000","last_change":"2024-09-14T04:01:59.052129+0000","last_active":"2024-09-14T04:14:09.069087+0000","last_peered":"2024-09-14T04:14:09.069087+0000","last_clean":"2024-09-14T04:14:09.069087+0000","last_became_active":"2024-09-14T04:01:59.051794+0000","last_became_peered":"2024-09-14T04:01:59.051794+0000","last_unstale":"2024-09-14T04:14:09.069087+0000","last_undegraded":"2024-09-14T04:14:09.069087+0000","last_fullsized":"2024-09-14T04:14:09.069087+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:38.687885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":387,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069935+0000","last_change":"2024-09-14T04:10:35.381626+0000","last_active":"2024-09-14T04:14:09.069935+0000","last_peered":"2024-09-14T04:14:09.069935+0000","last_clean":"2024-09-14T04:14:09.069935+0000","last_became_active":"2024-09-14T04:10:35.381351+0000","last_became_peered":"2024-09-14T04:10:35.381351+0000","last_unstale":"2024-09-14T04:14:09.069935+0000","last_undegraded":"2024-09-14T04:14:09.069935+0000","last_fullsized":"2024-09-14T04:14:09.069935+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.719069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"248'247","reported_seq":266,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.822843+0000","last_change":"2024-09-14T04:13:55.313340+0000","last_active":"2024-09-14T04:14:11.822843+0000","last_peered":"2024-09-14T04:14:11.822843+0000","last_clean":"2024-09-14T04:14:11.822843+0000","last_became_active":"2024-09-14T04:13:55.313042+0000","last_became_peered":"2024-09-14T04:13:55.313042+0000","last_unstale":"2024-09-14T04:14:11.822843+0000","last_undegraded":"2024-09-14T04:14:11.822843+0000","last_fullsized":"2024-09-14T04:14:11.822843+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":247,"log_dups_size":0,"ondisk_log_size":247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:39.539677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16187392,"num_objects":247,"num_object_clones":0,"num_object_copies":741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":247,"num_write_kb":15808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":487,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332027+0000","last_change":"2024-09-14T04:10:38.416435+0000","last_active":"2024-09-14T04:14:04.332027+0000","last_peered":"2024-09-14T04:14:04.332027+0000","last_clean":"2024-09-14T04:14:04.332027+0000","last_became_active":"2024-09-14T04:10:38.415996+0000","last_became_peered":"2024-09-14T04:10:38.415996+0000","last_unstale":"2024-09-14T04:14:04.332027+0000","last_undegraded":"2024-09-14T04:14:04.332027+0000","last_fullsized":"2024-09-14T04:14:04.332027+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:42.445222+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333171+0000","last_change":"2024-09-14T03:57:55.655109+0000","last_active":"2024-09-14T04:14:04.333171+0000","last_peered":"2024-09-14T04:14:04.333171+0000","last_clean":"2024-09-14T04:14:04.333171+0000","last_became_active":"2024-09-14T03:57:55.654640+0000","last_became_peered":"2024-09-14T03:57:55.654640+0000","last_unstale":"2024-09-14T04:14:04.333171+0000","last_undegraded":"2024-09-14T04:14:04.333171+0000","last_fullsized":"2024-09-14T04:14:04.333171+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:51.395202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368634+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:14:04.368634+0000","last_peered":"2024-09-14T04:14:04.368634+0000","last_clean":"2024-09-14T04:14:04.368634+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:14:04.368634+0000","last_undegraded":"2024-09-14T04:14:04.368634+0000","last_fullsized":"2024-09-14T04:14:04.368634+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332172+0000","last_change":"2024-09-14T04:06:46.336686+0000","last_active":"2024-09-14T04:14:04.332172+0000","last_peered":"2024-09-14T04:14:04.332172+0000","last_clean":"2024-09-14T04:14:04.332172+0000","last_became_active":"2024-09-14T04:06:46.336344+0000","last_became_peered":"2024-09-14T04:06:46.336344+0000","last_unstale":"2024-09-14T04:14:04.332172+0000","last_undegraded":"2024-09-14T04:14:04.332172+0000","last_fullsized":"2024-09-14T04:14:04.332172+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.186753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.d","version":"248'513","reported_seq":532,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:12.781785+0000","last_change":"2024-09-14T04:13:55.314198+0000","last_active":"2024-09-14T04:14:12.781785+0000","last_peered":"2024-09-14T04:14:12.781785+0000","last_clean":"2024-09-14T04:14:12.781785+0000","last_became_active":"2024-09-14T04:13:55.313909+0000","last_became_peered":"2024-09-14T04:13:55.313909+0000","last_unstale":"2024-09-14T04:14:12.781785+0000","last_undegraded":"2024-09-14T04:14:12.781785+0000","last_fullsized":"2024-09-14T04:14:12.781785+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":513,"log_dups_size":0,"ondisk_log_size":513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:21:37.634920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33619968,"num_objects":513,"num_object_clones":0,"num_object_copies":1539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":513,"num_write_kb":32832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":461,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333286+0000","last_change":"2024-09-14T04:08:36.230519+0000","last_active":"2024-09-14T04:14:04.333286+0000","last_peered":"2024-09-14T04:14:04.333286+0000","last_clean":"2024-09-14T04:14:04.333286+0000","last_became_active":"2024-09-14T04:08:31.385069+0000","last_became_peered":"2024-09-14T04:08:31.385069+0000","last_unstale":"2024-09-14T04:14:04.333286+0000","last_undegraded":"2024-09-14T04:14:04.333286+0000","last_fullsized":"2024-09-14T04:14:04.333286+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:08:36.230433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:02.161810+0000","scrub_duration":0.0020558019999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":257,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332903+0000","last_change":"2024-09-14T04:06:46.340884+0000","last_active":"2024-09-14T04:14:04.332903+0000","last_peered":"2024-09-14T04:14:04.332903+0000","last_clean":"2024-09-14T04:14:04.332903+0000","last_became_active":"2024-09-14T04:06:46.340193+0000","last_became_peered":"2024-09-14T04:06:46.340193+0000","last_unstale":"2024-09-14T04:14:04.332903+0000","last_undegraded":"2024-09-14T04:14:04.332903+0000","last_fullsized":"2024-09-14T04:14:04.332903+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:20.115136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":387,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069669+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:14:09.069669+0000","last_peered":"2024-09-14T04:14:09.069669+0000","last_clean":"2024-09-14T04:14:09.069669+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:14:09.069669+0000","last_undegraded":"2024-09-14T04:14:09.069669+0000","last_fullsized":"2024-09-14T04:14:09.069669+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":336,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069392+0000","last_change":"2024-09-14T04:11:13.271608+0000","last_active":"2024-09-14T04:14:09.069392+0000","last_peered":"2024-09-14T04:14:09.069392+0000","last_clean":"2024-09-14T04:14:09.069392+0000","last_became_active":"2024-09-14T04:11:13.271199+0000","last_became_peered":"2024-09-14T04:11:13.271199+0000","last_unstale":"2024-09-14T04:14:09.069392+0000","last_undegraded":"2024-09-14T04:14:09.069392+0000","last_fullsized":"2024-09-14T04:14:09.069392+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:31.977769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"248'816","reported_seq":835,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:14.975591+0000","last_change":"2024-09-14T04:13:55.312370+0000","last_active":"2024-09-14T04:14:14.975591+0000","last_peered":"2024-09-14T04:14:14.975591+0000","last_clean":"2024-09-14T04:14:14.975591+0000","last_became_active":"2024-09-14T04:13:55.312089+0000","last_became_peered":"2024-09-14T04:13:55.312089+0000","last_unstale":"2024-09-14T04:14:14.975591+0000","last_undegraded":"2024-09-14T04:14:14.975591+0000","last_fullsized":"2024-09-14T04:14:14.975591+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":816,"log_dups_size":0,"ondisk_log_size":816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:28.156275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":816,"num_write_kb":52224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":495,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332319+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:14:04.332319+0000","last_peered":"2024-09-14T04:14:04.332319+0000","last_clean":"2024-09-14T04:14:04.332319+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:14:04.332319+0000","last_undegraded":"2024-09-14T04:14:04.332319+0000","last_fullsized":"2024-09-14T04:14:04.332319+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":318,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069568+0000","last_change":"2024-09-14T04:01:59.055536+0000","last_active":"2024-09-14T04:14:09.069568+0000","last_peered":"2024-09-14T04:14:09.069568+0000","last_clean":"2024-09-14T04:14:09.069568+0000","last_became_active":"2024-09-14T04:01:59.055205+0000","last_became_peered":"2024-09-14T04:01:59.055205+0000","last_unstale":"2024-09-14T04:14:09.069568+0000","last_undegraded":"2024-09-14T04:14:09.069568+0000","last_fullsized":"2024-09-14T04:14:09.069568+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:40.874919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":264,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.369205+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:14:04.369205+0000","last_peered":"2024-09-14T04:14:04.369205+0000","last_clean":"2024-09-14T04:14:04.369205+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:14:04.369205+0000","last_undegraded":"2024-09-14T04:14:04.369205+0000","last_fullsized":"2024-09-14T04:14:04.369205+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341612+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:14:08.341612+0000","last_peered":"2024-09-14T04:14:08.341612+0000","last_clean":"2024-09-14T04:14:08.341612+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:14:08.341612+0000","last_undegraded":"2024-09-14T04:14:08.341612+0000","last_fullsized":"2024-09-14T04:14:08.341612+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"248'92","reported_seq":105,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:11.151864+0000","last_change":"2024-09-14T04:13:55.315605+0000","last_active":"2024-09-14T04:14:11.151864+0000","last_peered":"2024-09-14T04:14:11.151864+0000","last_clean":"2024-09-14T04:14:11.151864+0000","last_became_active":"2024-09-14T04:13:55.315334+0000","last_became_peered":"2024-09-14T04:13:55.315334+0000","last_unstale":"2024-09-14T04:14:11.151864+0000","last_undegraded":"2024-09-14T04:14:11.151864+0000","last_fullsized":"2024-09-14T04:14:11.151864+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:15.039826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":453,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368793+0000","last_change":"2024-09-14T04:08:33.947720+0000","last_active":"2024-09-14T04:14:04.368793+0000","last_peered":"2024-09-14T04:14:04.368793+0000","last_clean":"2024-09-14T04:14:04.368793+0000","last_became_active":"2024-09-14T04:08:31.385846+0000","last_became_peered":"2024-09-14T04:08:31.385846+0000","last_unstale":"2024-09-14T04:14:04.368793+0000","last_undegraded":"2024-09-14T04:14:04.368793+0000","last_fullsized":"2024-09-14T04:14:04.368793+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:47:06.972455+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":182,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.369500+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:14:04.369500+0000","last_peered":"2024-09-14T04:14:04.369500+0000","last_clean":"2024-09-14T04:14:04.369500+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:14:04.369500+0000","last_undegraded":"2024-09-14T04:14:04.369500+0000","last_fullsized":"2024-09-14T04:14:04.369500+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":319,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.070180+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:14:09.070180+0000","last_peered":"2024-09-14T04:14:09.070180+0000","last_clean":"2024-09-14T04:14:09.070180+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:14:09.070180+0000","last_undegraded":"2024-09-14T04:14:09.070180+0000","last_fullsized":"2024-09-14T04:14:09.070180+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368167+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:14:04.368167+0000","last_peered":"2024-09-14T04:14:04.368167+0000","last_clean":"2024-09-14T04:14:04.368167+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:14:04.368167+0000","last_undegraded":"2024-09-14T04:14:04.368167+0000","last_fullsized":"2024-09-14T04:14:04.368167+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"248'17","reported_seq":30,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:10.882596+0000","last_change":"2024-09-14T04:13:55.312894+0000","last_active":"2024-09-14T04:14:10.882596+0000","last_peered":"2024-09-14T04:14:10.882596+0000","last_clean":"2024-09-14T04:14:10.882596+0000","last_became_active":"2024-09-14T04:13:55.312568+0000","last_became_peered":"2024-09-14T04:13:55.312568+0000","last_unstale":"2024-09-14T04:14:10.882596+0000","last_undegraded":"2024-09-14T04:14:10.882596+0000","last_fullsized":"2024-09-14T04:14:10.882596+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:13:54.293151+0000","last_clean_scrub_stamp":"2024-09-14T04:13:54.293151+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:30.428884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17,"num_write_kb":1088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":544,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069249+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:14:09.069249+0000","last_peered":"2024-09-14T04:14:09.069249+0000","last_clean":"2024-09-14T04:14:09.069249+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:14:09.069249+0000","last_undegraded":"2024-09-14T04:14:09.069249+0000","last_fullsized":"2024-09-14T04:14:09.069249+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":93,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341561+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:14:08.341561+0000","last_peered":"2024-09-14T04:14:08.341561+0000","last_clean":"2024-09-14T04:14:08.341561+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:14:08.341561+0000","last_undegraded":"2024-09-14T04:14:08.341561+0000","last_fullsized":"2024-09-14T04:14:08.341561+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":247,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333386+0000","last_change":"2024-09-14T04:09:50.221627+0000","last_active":"2024-09-14T04:14:04.333386+0000","last_peered":"2024-09-14T04:14:04.333386+0000","last_clean":"2024-09-14T04:14:04.333386+0000","last_became_active":"2024-09-14T04:09:50.220752+0000","last_became_peered":"2024-09-14T04:09:50.220752+0000","last_unstale":"2024-09-14T04:14:04.333386+0000","last_undegraded":"2024-09-14T04:14:04.333386+0000","last_fullsized":"2024-09-14T04:14:04.333386+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:51.513473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368415+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:14:04.368415+0000","last_peered":"2024-09-14T04:14:04.368415+0000","last_clean":"2024-09-14T04:14:04.368415+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:14:04.368415+0000","last_undegraded":"2024-09-14T04:14:04.368415+0000","last_fullsized":"2024-09-14T04:14:04.368415+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368502+0000","last_change":"2024-09-14T04:01:59.055488+0000","last_active":"2024-09-14T04:14:04.368502+0000","last_peered":"2024-09-14T04:14:04.368502+0000","last_clean":"2024-09-14T04:14:04.368502+0000","last_became_active":"2024-09-14T04:01:59.055079+0000","last_became_peered":"2024-09-14T04:01:59.055079+0000","last_unstale":"2024-09-14T04:14:04.368502+0000","last_undegraded":"2024-09-14T04:14:04.368502+0000","last_fullsized":"2024-09-14T04:14:04.368502+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:43.903778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368777+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:14:04.368777+0000","last_peered":"2024-09-14T04:14:04.368777+0000","last_clean":"2024-09-14T04:14:04.368777+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:14:04.368777+0000","last_undegraded":"2024-09-14T04:14:04.368777+0000","last_fullsized":"2024-09-14T04:14:04.368777+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":319,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.069279+0000","last_change":"2024-09-14T04:01:59.055798+0000","last_active":"2024-09-14T04:14:09.069279+0000","last_peered":"2024-09-14T04:14:09.069279+0000","last_clean":"2024-09-14T04:14:09.069279+0000","last_became_active":"2024-09-14T04:01:59.055401+0000","last_became_peered":"2024-09-14T04:01:59.055401+0000","last_unstale":"2024-09-14T04:14:09.069279+0000","last_undegraded":"2024-09-14T04:14:09.069279+0000","last_fullsized":"2024-09-14T04:14:09.069279+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:31.732518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":256,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.369036+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:14:04.369036+0000","last_peered":"2024-09-14T04:14:04.369036+0000","last_clean":"2024-09-14T04:14:04.369036+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:14:04.369036+0000","last_undegraded":"2024-09-14T04:14:04.369036+0000","last_fullsized":"2024-09-14T04:14:04.369036+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332230+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:14:04.332230+0000","last_peered":"2024-09-14T04:14:04.332230+0000","last_clean":"2024-09-14T04:14:04.332230+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:14:04.332230+0000","last_undegraded":"2024-09-14T04:14:04.332230+0000","last_fullsized":"2024-09-14T04:14:04.332230+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":179,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368783+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:14:04.368783+0000","last_peered":"2024-09-14T04:14:04.368783+0000","last_clean":"2024-09-14T04:14:04.368783+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:14:04.368783+0000","last_undegraded":"2024-09-14T04:14:04.368783+0000","last_fullsized":"2024-09-14T04:14:04.368783+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341795+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:14:08.341795+0000","last_peered":"2024-09-14T04:14:08.341795+0000","last_clean":"2024-09-14T04:14:08.341795+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:14:08.341795+0000","last_undegraded":"2024-09-14T04:14:08.341795+0000","last_fullsized":"2024-09-14T04:14:08.341795+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":197,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.068401+0000","last_change":"2024-09-14T04:06:46.344370+0000","last_active":"2024-09-14T04:14:09.068401+0000","last_peered":"2024-09-14T04:14:09.068401+0000","last_clean":"2024-09-14T04:14:09.068401+0000","last_became_active":"2024-09-14T04:06:46.343836+0000","last_became_peered":"2024-09-14T04:06:46.343836+0000","last_unstale":"2024-09-14T04:14:09.068401+0000","last_undegraded":"2024-09-14T04:14:09.068401+0000","last_fullsized":"2024-09-14T04:14:09.068401+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:56.671042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.333347+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:14:04.333347+0000","last_peered":"2024-09-14T04:14:04.333347+0000","last_clean":"2024-09-14T04:14:04.333347+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:14:04.333347+0000","last_undegraded":"2024-09-14T04:14:04.333347+0000","last_fullsized":"2024-09-14T04:14:04.333347+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341149+0000","last_change":"2024-09-14T04:10:35.382082+0000","last_active":"2024-09-14T04:14:08.341149+0000","last_peered":"2024-09-14T04:14:08.341149+0000","last_clean":"2024-09-14T04:14:08.341149+0000","last_became_active":"2024-09-14T04:10:35.381070+0000","last_became_peered":"2024-09-14T04:10:35.381070+0000","last_unstale":"2024-09-14T04:14:08.341149+0000","last_undegraded":"2024-09-14T04:14:08.341149+0000","last_fullsized":"2024-09-14T04:14:08.341149+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:03.698151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341569+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:14:08.341569+0000","last_peered":"2024-09-14T04:14:08.341569+0000","last_clean":"2024-09-14T04:14:08.341569+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:14:08.341569+0000","last_undegraded":"2024-09-14T04:14:08.341569+0000","last_fullsized":"2024-09-14T04:14:08.341569+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":345,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:09.068973+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:14:09.068973+0000","last_peered":"2024-09-14T04:14:09.068973+0000","last_clean":"2024-09-14T04:14:09.068973+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:14:09.068973+0000","last_undegraded":"2024-09-14T04:14:09.068973+0000","last_fullsized":"2024-09-14T04:14:09.068973+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":169,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332740+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:14:04.332740+0000","last_peered":"2024-09-14T04:14:04.332740+0000","last_clean":"2024-09-14T04:14:04.332740+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:14:04.332740+0000","last_undegraded":"2024-09-14T04:14:04.332740+0000","last_fullsized":"2024-09-14T04:14:04.332740+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":243,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341380+0000","last_change":"2024-09-14T04:08:15.321997+0000","last_active":"2024-09-14T04:14:08.341380+0000","last_peered":"2024-09-14T04:14:08.341380+0000","last_clean":"2024-09-14T04:14:08.341380+0000","last_became_active":"2024-09-14T04:08:15.320268+0000","last_became_peered":"2024-09-14T04:08:15.320268+0000","last_unstale":"2024-09-14T04:14:08.341380+0000","last_undegraded":"2024-09-14T04:14:08.341380+0000","last_fullsized":"2024-09-14T04:14:08.341380+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:31.799666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":361,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.670776+0000","last_change":"2024-09-14T04:06:46.338928+0000","last_active":"2024-09-14T04:14:08.670776+0000","last_peered":"2024-09-14T04:14:08.670776+0000","last_clean":"2024-09-14T04:14:08.670776+0000","last_became_active":"2024-09-14T04:06:46.338615+0000","last_became_peered":"2024-09-14T04:06:46.338615+0000","last_unstale":"2024-09-14T04:14:08.670776+0000","last_undegraded":"2024-09-14T04:14:08.670776+0000","last_fullsized":"2024-09-14T04:14:08.670776+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:22.068274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":89,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341513+0000","last_change":"2024-09-14T04:10:35.387036+0000","last_active":"2024-09-14T04:14:08.341513+0000","last_peered":"2024-09-14T04:14:08.341513+0000","last_clean":"2024-09-14T04:14:08.341513+0000","last_became_active":"2024-09-14T04:10:35.386675+0000","last_became_peered":"2024-09-14T04:10:35.386675+0000","last_unstale":"2024-09-14T04:14:08.341513+0000","last_undegraded":"2024-09-14T04:14:08.341513+0000","last_fullsized":"2024-09-14T04:14:08.341513+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:45.844989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":39,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368310+0000","last_change":"2024-09-14T04:11:13.271013+0000","last_active":"2024-09-14T04:14:04.368310+0000","last_peered":"2024-09-14T04:14:04.368310+0000","last_clean":"2024-09-14T04:14:04.368310+0000","last_became_active":"2024-09-14T04:11:13.270613+0000","last_became_peered":"2024-09-14T04:11:13.270613+0000","last_unstale":"2024-09-14T04:14:04.368310+0000","last_undegraded":"2024-09-14T04:14:04.368310+0000","last_fullsized":"2024-09-14T04:14:04.368310+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:13.412234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":255,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332392+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:14:04.332392+0000","last_peered":"2024-09-14T04:14:04.332392+0000","last_clean":"2024-09-14T04:14:04.332392+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:14:04.332392+0000","last_undegraded":"2024-09-14T04:14:04.332392+0000","last_fullsized":"2024-09-14T04:14:04.332392+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":39,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.368326+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:14:04.368326+0000","last_peered":"2024-09-14T04:14:04.368326+0000","last_clean":"2024-09-14T04:14:04.368326+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:14:04.368326+0000","last_undegraded":"2024-09-14T04:14:04.368326+0000","last_fullsized":"2024-09-14T04:14:04.368326+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":242,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332905+0000","last_change":"2024-09-14T04:08:15.322439+0000","last_active":"2024-09-14T04:14:04.332905+0000","last_peered":"2024-09-14T04:14:04.332905+0000","last_clean":"2024-09-14T04:14:04.332905+0000","last_became_active":"2024-09-14T04:08:15.322089+0000","last_became_peered":"2024-09-14T04:08:15.322089+0000","last_unstale":"2024-09-14T04:14:04.332905+0000","last_undegraded":"2024-09-14T04:14:04.332905+0000","last_fullsized":"2024-09-14T04:14:04.332905+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:54.726700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":323,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.670378+0000","last_change":"2024-09-14T04:08:16.316538+0000","last_active":"2024-09-14T04:14:08.670378+0000","last_peered":"2024-09-14T04:14:08.670378+0000","last_clean":"2024-09-14T04:14:08.670378+0000","last_became_active":"2024-09-14T04:08:16.316210+0000","last_became_peered":"2024-09-14T04:08:16.316210+0000","last_unstale":"2024-09-14T04:14:08.670378+0000","last_undegraded":"2024-09-14T04:14:08.670378+0000","last_fullsized":"2024-09-14T04:14:08.670378+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:21.383460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":124,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:04.332713+0000","last_change":"2024-09-14T04:07:20.032386+0000","last_active":"2024-09-14T04:14:04.332713+0000","last_peered":"2024-09-14T04:14:04.332713+0000","last_clean":"2024-09-14T04:14:04.332713+0000","last_became_active":"2024-09-14T04:07:20.032017+0000","last_became_peered":"2024-09-14T04:07:20.032017+0000","last_unstale":"2024-09-14T04:14:04.332713+0000","last_undegraded":"2024-09-14T04:14:04.332713+0000","last_fullsized":"2024-09-14T04:14:04.332713+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:50.612390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-09-14T04:14:05.891361+0000","last_change":"2024-09-14T03:57:55.681453+0000","last_active":"2024-09-14T04:14:05.891361+0000","last_peered":"2024-09-14T04:14:05.891361+0000","last_clean":"2024-09-14T04:14:05.891361+0000","last_became_active":"2024-09-14T03:57:55.681119+0000","last_became_peered":"2024-09-14T03:57:55.681119+0000","last_unstale":"2024-09-14T04:14:05.891361+0000","last_undegraded":"2024-09-14T04:14:05.891361+0000","last_fullsized":"2024-09-14T04:14:05.891361+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:00.618476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":95,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-09-14T04:14:08.341779+0000","last_change":"2024-09-14T04:10:35.400336+0000","last_active":"2024-09-14T04:14:08.341779+0000","last_peered":"2024-09-14T04:14:08.341779+0000","last_clean":"2024-09-14T04:14:08.341779+0000","last_became_active":"2024-09-14T04:10:35.383628+0000","last_became_peered":"2024-09-14T04:10:35.383628+0000","last_unstale":"2024-09-14T04:14:08.341779+0000","last_undegraded":"2024-09-14T04:14:08.341779+0000","last_fullsized":"2024-09-14T04:14:08.341779+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:35.608521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4999,"num_write_kb":319936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1164967936,"data_stored":1164967936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4999,"ondisk_log_size":4999,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":946,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201683,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":800988,"kb_used_data":3664,"kb_used_omap":1,"kb_used_meta":797310,"kb_avail":92944164,"statfs":{"total":95995035648,"available":95174823936,"internally_reserved":0,"allocated":3751936,"data_stored":2178135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":816445899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463321,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691980,"kb_used_data":29188,"kb_used_omap":1,"kb_used_meta":662782,"kb_avail":93053172,"statfs":{"total":95995035648,"available":95286448128,"internally_reserved":0,"allocated":29888512,"data_stored":28326352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":678689223},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724957,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070796,"kb_used_data":42948,"kb_used_omap":1,"kb_used_meta":1027838,"kb_avail":92674356,"statfs":{"total":95995035648,"available":94898540544,"internally_reserved":0,"allocated":43978752,"data_stored":42413235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1052506566},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986593,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850712,"kb_used_data":3340,"kb_used_omap":1,"kb_used_meta":847358,"kb_avail":92894440,"statfs":{"total":95995035648,"available":95123906560,"internally_reserved":0,"allocated":3420160,"data_stored":1850146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":867695046},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748622,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":444296,"kb_used_data":31872,"kb_used_omap":4,"kb_used_meta":412411,"kb_avail":93300856,"statfs":{"total":95995035648,"available":95540076544,"internally_reserved":0,"allocated":32636928,"data_stored":31075219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":422309420},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912149,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":735820,"kb_used_data":404548,"kb_used_omap":6,"kb_used_meta":331257,"kb_avail":93009332,"statfs":{"total":95995035648,"available":95241555968,"internally_reserved":0,"allocated":414257152,"data_stored":412691873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":339207737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468166,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937536,"kb_used_data":423864,"kb_used_omap":5,"kb_used_meta":513658,"kb_avail":92807616,"statfs":{"total":95995035648,"available":95034998784,"internally_reserved":0,"allocated":434036736,"data_stored":432481184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":525986143},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342281,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410700,"kb_used_data":194756,"kb_used_omap":10,"kb_used_meta":215925,"kb_avail":93334452,"statfs":{"total":95995035648,"available":95574478848,"internally_reserved":0,"allocated":199430144,"data_stored":197864343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":221107410},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":214827008,"data_stored":214827008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":448724992,"data_stored":448724992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":184090624,"data_stored":184090624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":115212288,"data_stored":115212288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8847360,"data_stored":8847360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":73793536,"data_stored":73793536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":82771968,"data_stored":82771968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":36700160,"data_stored":36700160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:14:18.078 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:14:18.078 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:14:18.078 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-14T04:14:18.078 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:14:18.078 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:14:18.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-14T04:14:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:18.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:18.194 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:18.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:18.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.385 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:18.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:18.427 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:14:18.486 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:18.525 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:18.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:18.666 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[107399]: pgmap v870: 99 pgs: 99 active+clean; 959 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[112165]: pgmap v870: 99 pgs: 99 active+clean; 959 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:18.765 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 23533 23517 183.682 151.938 0.00395478 0.00543857 2024-09-14T04:14:18.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:18.812 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:18 smithi012 ceph-mon[101180]: pgmap v870: 99 pgs: 99 active+clean; 959 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:14:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:18.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:18.949 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:18.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:19.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:19.072 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:19.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:19.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:19.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:19.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:19.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:19.472 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:19.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:19.611 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:19.712 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:19.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:19.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:19.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 25477 25461 176.773 121.5 0.0088075 0.00565198 2024-09-14T04:14:19.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:19.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:19.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:19.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:20.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:20.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:20.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:20.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:20.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:20.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:20.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:20.385 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:20.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:20.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:20.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:20.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:20.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:20.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[107399]: pgmap v871: 99 pgs: 99 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:14:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[112165]: pgmap v871: 99 pgs: 99 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:14:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:20.725 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:20 smithi012 ceph-mon[101180]: pgmap v871: 99 pgs: 99 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-14T04:14:20.725 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:20.725 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:20.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:20.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 26883 26867 167.883 87.875 0.00724829 0.00595093 2024-09-14T04:14:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:20.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:20.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:20.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:20.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:21.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:21.080 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:21.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:21.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:21.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:21.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:21.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:21.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:21.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:21.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:21.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:21.754 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:21.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 28977 28961 164.518 130.875 0.0040489 0.00607348 2024-09-14T04:14:21.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:21.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:21.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:21.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:22.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:22.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:22.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:22.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:22.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:22.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:22.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:22.550 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:22.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:22.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:22.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:22 smithi001 ceph-mon[107399]: pgmap v872: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:22 smithi001 ceph-mon[112165]: pgmap v872: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:22.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:22.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:22.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 30997 30981 161.328 126.25 0.0042672 0.00619367 2024-09-14T04:14:22.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:22 smithi012 ceph-mon[101180]: pgmap v872: 99 pgs: 99 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:22.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:22.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:22.998 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:22.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:22.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:23.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:14:23.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:23.134 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:23.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.278 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:23.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:23.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:23.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:23.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:23.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:23.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:23.767 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 33008 32992 158.586 125.688 0.0164955 0.00630076 2024-09-14T04:14:23.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:23.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:23.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:23.957 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:23.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:23.983 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:14:23.984 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2024-09-14T04:14:23.984 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:14:24.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:24.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:24.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:24.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:24.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:24.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:24.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:24.404 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:14:24.434 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[112165]: pgmap v873: 99 pgs: 99 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-14T04:14:24.434 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2426761433' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.434 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.434 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[107399]: pgmap v873: 99 pgs: 99 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-14T04:14:24.435 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2426761433' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.435 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:24 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:24.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:24.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:24.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:24.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:24.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:24.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:24.767 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 34745 34729 155.013 108.562 0.00950838 0.00644496 2024-09-14T04:14:24.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:24 smithi012 ceph-mon[101180]: pgmap v873: 99 pgs: 99 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-14T04:14:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2426761433' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:24 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:24.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:24.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:24.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:14:24.975 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:14:24.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:14:25.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:25.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:25.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:25.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:25.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:25.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:25.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:25.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:25.454 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:25.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:25.603 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:25.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[112165]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4257246469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[107399]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T04:14:25.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4257246469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:25.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:25.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 36759 36743 153.069 125.875 0.00959896 0.00652784 2024-09-14T04:14:25.800 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:25 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:25.800 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:25 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:25.800 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:25 smithi012 ceph-mon[101180]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T04:14:25.800 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4257246469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:25.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:25.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:25.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:25.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:26.026 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:26.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:26.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:26.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:26.310 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:26.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:26.448 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:26.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:26.583 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:26.627 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:26.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:26 smithi001 ceph-mon[112165]: pgmap v875: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-14T04:14:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:26 smithi001 ceph-mon[107399]: pgmap v875: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-14T04:14:26.704 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:26] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-09-14T04:14:26.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:26.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 38873 38857 151.759 132.125 0.00575821 0.00658443 2024-09-14T04:14:26.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:26.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:26 smithi012 ceph-mon[101180]: pgmap v875: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-14T04:14:26.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:26.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:26.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:26.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:27.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:27.127 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:27.229 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:27.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:27.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:27.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:27.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:27.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:27.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:27.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:27.766 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 40920 40904 150.358 127.938 0.00520565 0.00664568 2024-09-14T04:14:27.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:27.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:27.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:27.956 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:27.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:28.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:14:28.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:28.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:28.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:28.211 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:28.313 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:28.351 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:28.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:28.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:28.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:28.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:14:28.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:28 smithi001 ceph-mon[107399]: pgmap v876: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:14:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:28 smithi001 ceph-mon[112165]: pgmap v876: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:14:28.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:28.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:28.767 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 42648 42632 147.996 108 0.0134666 0.00675085 2024-09-14T04:14:28.775 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:28 smithi012 ceph-mon[101180]: pgmap v876: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:14:28.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:28.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:28.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:28.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:28.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:29.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:29.132 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:29.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:29.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:29.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:29.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:29.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:29.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:29.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:29.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:29.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:29.685 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:29.692 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:14:29.692 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:14:29.722 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:29.767 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 44485 44469 146.249 114.812 0.00417052 0.00683289 2024-09-14T04:14:29.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:29.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:29.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:29.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:29.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:30.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:30.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:30.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:30.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:30.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:30.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.403 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:30.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:30.504 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:30.540 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:30.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:30.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:30.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:30 smithi001 ceph-mon[107399]: pgmap v877: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:14:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:30 smithi001 ceph-mon[112165]: pgmap v877: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:14:30.768 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:14:30.767620+0000 min lat: 0.00131144 max lat: 0.190069 avg lat: 0.00689745 2024-09-14T04:14:30.768 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:14:30.768 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 46384 46368 144.87 118.688 0.00776372 0.00689745 2024-09-14T04:14:30.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:30.803 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:30 smithi012 ceph-mon[101180]: pgmap v877: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:14:30.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:30.938 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:30.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:30.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:31.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:31.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:31.156 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:14:31.156 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:14:31.157 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.5211212078747245 2024-09-14T04:14:31.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 6 0.5211212078747245 2024-09-14T04:14:31.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:31.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:31.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:31.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:31.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:31.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:31.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:31.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:31.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:31.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:31.768 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 48034 48018 142.882 103.125 0.00313582 0.00699295 2024-09-14T04:14:31.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:31.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:31.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:31.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:31.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:32.015 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:32.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:32.154 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:32.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:32.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:32.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:32.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:32.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:32.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:32.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:32.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:32.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:32.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:32 smithi001 ceph-mon[107399]: pgmap v878: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T04:14:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:32 smithi001 ceph-mon[112165]: pgmap v878: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T04:14:32.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 49746 49730 141.246 107 0.00517792 0.00707409 2024-09-14T04:14:32.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:32 smithi012 ceph-mon[101180]: pgmap v878: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-14T04:14:32.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:32.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:32.975 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:33.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:33.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:33.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:33.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:33.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:33.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:33.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:33.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:33.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:33.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:33.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:33.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 51586 51570 140.105 115 0.00517477 0.00713301 2024-09-14T04:14:33.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:33.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:14:33.842 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:33.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:33.995 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:33.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:33.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:34.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:34.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:34.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:34.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:34.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:34.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:34.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:34.506 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:34.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:34.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:34.593 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:34.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:34.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:34.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:34 smithi001 ceph-mon[107399]: pgmap v879: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:14:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:34 smithi001 ceph-mon[112165]: pgmap v879: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:14:34.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 53110 53094 138.236 95.25 0.0170804 0.00722898 2024-09-14T04:14:34.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:34 smithi012 ceph-mon[101180]: pgmap v879: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-14T04:14:34.857 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:34.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:34.959 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:35.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:35.032 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:35.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:35.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:35.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:35.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:35.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:35.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:35.453 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:14:35.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.553 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:35.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:35.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:35.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:35.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2204031218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2204031218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:35.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 54775 54759 136.869 104.062 0.00822807 0.00730098 2024-09-14T04:14:35.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:35.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2204031218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:35 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:14:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:35.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:35.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:35.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:36.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:36.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:36.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:36.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:36.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:36.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:36.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:36.515 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:36.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:36.652 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:36.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:36.664 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[107399]: pgmap v880: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:36.664 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:36.664 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:14:36.664 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[107399]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T04:14:36.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[112165]: pgmap v880: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:36.665 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:36.665 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:14:36.665 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-mon[112165]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T04:14:36.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:36 smithi012 ceph-mon[101180]: pgmap v880: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:14:36.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:36 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:14:36.748 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:36 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:14:36.748 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:36 smithi012 ceph-mon[101180]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T04:14:36.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:36.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 56500 56484 135.751 107.812 0.0105492 0.0073608 2024-09-14T04:14:36.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:36.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:36.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:36.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:36] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-09-14T04:14:37.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:37.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:37.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:37.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:37.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:37.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:37.442 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:37.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:37.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:37.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:37.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:37.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:37.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:37.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:37.769 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 58069 58053 134.354 98.0625 0.00865144 0.0074375 2024-09-14T04:14:37.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:37.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:37.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:37.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:38.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:38.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:38.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:38.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:38.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:38.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:38.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:38.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:14:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:38.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:38.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:38.604 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:38.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:38.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:38 smithi001 ceph-mon[112165]: pgmap v882: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-14T04:14:38.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:38 smithi001 ceph-mon[107399]: pgmap v882: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-14T04:14:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:38.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:38.770 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 59471 59455 132.682 87.625 0.00470981 0.0075308 2024-09-14T04:14:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:38 smithi012 ceph-mon[101180]: pgmap v882: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-14T04:14:38.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:38.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:38.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:38.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:39.004 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:39.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:39.140 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:39.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:39.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:39.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:39.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:39.504 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:39.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:39.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:39.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:39.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:39.770 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 60802 60786 130.975 83.1875 0.0087503 0.00763054 2024-09-14T04:14:39.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:39.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:39.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:39 smithi001 ceph-mon[107399]: pgmap v883: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T04:14:39.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:39 smithi001 ceph-mon[112165]: pgmap v883: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T04:14:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:39 smithi012 ceph-mon[101180]: pgmap v883: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-14T04:14:39.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:39.954 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:39.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:40.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:40.095 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:40.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:40.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:40.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:40.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:40.368 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:40.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.513 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:40.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:40.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:40.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:40.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:40.770 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 62164 62148 129.447 85.125 0.0156248 0.00772071 2024-09-14T04:14:40.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:40.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:40.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:40.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:41.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:41.074 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:41.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.222 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:41.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:41.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.355 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:41.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:41.456 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:41.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:41.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:41.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:41.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:41.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:41.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:41.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 63347 63331 127.653 73.9375 0.00549727 0.00782785 2024-09-14T04:14:41.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:41.897 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:41.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:41.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:41.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:42.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:42.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:42.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:42.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:42.225 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:42.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:42.300 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.6 to 0.521121 (8568) 2024-09-14T04:14:42.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:42.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:42.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[107399]: pgmap v884: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/914605535' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[112165]: pgmap v884: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/914605535' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:42.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:42.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:42 smithi012 ceph-mon[101180]: pgmap v884: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:14:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/914605535' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245} : dispatch 2024-09-14T04:14:42.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:42.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:42.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 64172 64156 125.276 51.5625 0.00260596 0.00787507 2024-09-14T04:14:42.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:42.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:42.867 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:14:42.867 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:14:42.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:42.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:42.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:43.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:43.057 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:43.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:43.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:43.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:43.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:43.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:43.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:43.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245}]': finished 2024-09-14T04:14:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[107399]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T04:14:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[107399]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T04:14:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245}]': finished 2024-09-14T04:14:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[112165]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T04:14:43.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:43 smithi001 ceph-mon[112165]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T04:14:43.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:43.568 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:43 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5211212078747245}]': finished 2024-09-14T04:14:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:43 smithi012 ceph-mon[101180]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T04:14:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:43 smithi012 ceph-mon[101180]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T04:14:43.670 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:43.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:43.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 64172 64156 121.48 0 - 0.00787507 2024-09-14T04:14:43.797 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:43.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:43.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:43.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:43.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:43.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:44.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:44.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:44.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:44.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:44.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:44.332 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:44.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:44.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[112165]: pgmap v886: 99 pgs: 1 peering, 98 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[112165]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[107399]: pgmap v886: 99 pgs: 1 peering, 98 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:14:44.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:44 smithi001 ceph-mon[107399]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T04:14:44.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:44 smithi012 ceph-mon[101180]: pgmap v886: 99 pgs: 1 peering, 98 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-14T04:14:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:44 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:14:44.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:44 smithi012 ceph-mon[101180]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T04:14:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.603 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:44.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:44.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:44.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:44.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 64548 64532 118.599 11.75 0.00843533 0.00842122 2024-09-14T04:14:44.817 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:14:44.817 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:14:44.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:44.851 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:14:44.852 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:14:44.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:44.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:44.884 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:14:44.885 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:14:44.923 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:14:44.923 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:14:44.955 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:14:44.956 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:14:44.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:44.996 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:14:44.997 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:14:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:45.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:45.031 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:14:45.031 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:14:45.069 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:14:45.069 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:14:45.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:45.149 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:45.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:45.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:45.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:45.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:45.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:45.575 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:45.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/501936849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:45.677 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:45.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/501936849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/501936849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:45.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 65795 65779 117.437 77.9375 0.0118109 0.00851018 2024-09-14T04:14:45.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:45.864 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:45.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:45.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:45.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:45.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:46.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:46.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:46.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:46.254 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:46.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:46.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:46.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:46.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:46.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:46 smithi012 ceph-mon[101180]: pgmap v889: 99 pgs: 1 active+undersized+remapped, 1 peering, 97 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-14T04:14:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:46 smithi012 ceph-mon[101180]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T04:14:46.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:46.658 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:46.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:46 smithi001 ceph-mon[112165]: pgmap v889: 99 pgs: 1 active+undersized+remapped, 1 peering, 97 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-14T04:14:46.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:46 smithi001 ceph-mon[112165]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T04:14:46.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:46 smithi001 ceph-mon[107399]: pgmap v889: 99 pgs: 1 active+undersized+remapped, 1 peering, 97 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-14T04:14:46.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:46 smithi001 ceph-mon[107399]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T04:14:46.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:46.771 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 67001 66985 116.268 75.375 0.00919206 0.00859546 2024-09-14T04:14:46.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:46.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:46.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:46.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:46.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:46] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-09-14T04:14:47.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:47.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:14:47.064 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:47.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:47.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:47.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:47.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:47.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:47.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:47.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:47.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:47.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:47.664 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:47 smithi001 ceph-mon[107399]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T04:14:47.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:47 smithi001 ceph-mon[112165]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T04:14:47.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:47.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:47.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:47.772 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 68193 68177 115.138 74.5 0.00516292 0.00867752 2024-09-14T04:14:47.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:47 smithi012 ceph-mon[101180]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T04:14:47.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:47.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:47.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:48.021 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:48.123 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:48.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:48.261 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:48.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:48.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:48.429 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:48.458 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[112165]: 1.7 scrub starts 2024-09-14T04:14:48.458 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[112165]: 1.7 scrub ok 2024-09-14T04:14:48.458 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[112165]: pgmap v892: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering, 94 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 50 MiB/s wr, 804 op/s; 3937/197682 objects degraded (1.992%); 3961/197682 objects misplaced (2.004%); 823 KiB/s, 12 objects/s recovering 2024-09-14T04:14:48.458 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 3937/197682 objects degraded (1.992%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:48.459 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[107399]: 1.7 scrub starts 2024-09-14T04:14:48.459 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[107399]: 1.7 scrub ok 2024-09-14T04:14:48.459 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[107399]: pgmap v892: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering, 94 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 50 MiB/s wr, 804 op/s; 3937/197682 objects degraded (1.992%); 3961/197682 objects misplaced (2.004%); 823 KiB/s, 12 objects/s recovering 2024-09-14T04:14:48.459 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:48 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 3937/197682 objects degraded (1.992%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:48.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:48.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:48 smithi012 ceph-mon[101180]: 1.7 scrub starts 2024-09-14T04:14:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:48 smithi012 ceph-mon[101180]: 1.7 scrub ok 2024-09-14T04:14:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:48 smithi012 ceph-mon[101180]: pgmap v892: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering, 94 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 50 MiB/s wr, 804 op/s; 3937/197682 objects degraded (1.992%); 3961/197682 objects misplaced (2.004%); 823 KiB/s, 12 objects/s recovering 2024-09-14T04:14:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:48 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 3937/197682 objects degraded (1.992%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:48.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:48.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:48.772 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 69419 69403 114.124 76.625 0.0103458 0.00875732 2024-09-14T04:14:48.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:48.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:48.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:48.986 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:48.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:48.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:48.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:49.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:49.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:49.183 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:14:49.183 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:14:49.183 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-14T04:14:49.183 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:14:49.183 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:14:49.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-14T04:14:49.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:49.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:49.409 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.459 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:49.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:49.560 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:49.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:49.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:49 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:14:49.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:49.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:49 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:14:49.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:49.691 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:49.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:49.772 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 70510 70494 112.947 68.1875 0.00894276 0.00884846 2024-09-14T04:14:49.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:49 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:14:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:14:49.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:49.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:49.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:50.001 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:50.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:50.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:50.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:50.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:14:50.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.269 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:50.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:50.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:50.372 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:14:50.396 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:50.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:50.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:50.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 42 MiB/s wr, 687 op/s; 3937/200277 objects degraded (1.966%); 3961/200277 objects misplaced (1.978%); 554 KiB/s, 8 objects/s recovering 2024-09-14T04:14:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[107399]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 42 MiB/s wr, 687 op/s; 3937/200277 objects degraded (1.966%); 3961/200277 objects misplaced (1.978%); 554 KiB/s, 8 objects/s recovering 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:50.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:50 smithi001 ceph-mon[112165]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T04:14:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:50.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:50.773 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:14:50.772927+0000 min lat: 0.00131144 max lat: 2.13766 avg lat: 0.0089317 2024-09-14T04:14:50.773 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:14:50.773 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 71648 71632 111.899 71.125 0.0124337 0.0089317 2024-09-14T04:14:50.824 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 42 MiB/s wr, 687 op/s; 3937/200277 objects degraded (1.966%); 3961/200277 objects misplaced (1.978%); 554 KiB/s, 8 objects/s recovering 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2893071921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:14:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:50 smithi012 ceph-mon[101180]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T04:14:50.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:50.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:50.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:51.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:51.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:51.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:51.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:51.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:51.376 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:51.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:51.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:51.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:51.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:51.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:51.677 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:51.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:51.755 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:51.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:51.774 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 72746 72730 110.841 68.625 0.0080441 0.0090164 2024-09-14T04:14:51.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:51.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:51.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:51.994 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:52.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:52.108 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:14:52.108 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:14:52.108 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:14:52.108 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:14:52.108 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:14:52.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:52.157 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:52.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:52.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:52.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:52.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:52.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:52.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:52.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:52.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:52.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:52.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:52 smithi001 ceph-mon[112165]: pgmap v895: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.00k op/s; 3937/208230 objects degraded (1.891%); 3961/208230 objects misplaced (1.902%); 548 KiB/s, 8 objects/s recovering 2024-09-14T04:14:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:52 smithi001 ceph-mon[107399]: pgmap v895: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.00k op/s; 3937/208230 objects degraded (1.891%); 3961/208230 objects misplaced (1.902%); 548 KiB/s, 8 objects/s recovering 2024-09-14T04:14:52.774 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 73875 73859 109.882 70.5625 0.00499872 0.00909301 2024-09-14T04:14:52.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.843 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:52.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:52 smithi012 ceph-mon[101180]: pgmap v895: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 62 MiB/s wr, 1.00k op/s; 3937/208230 objects degraded (1.891%); 3961/208230 objects misplaced (1.902%); 548 KiB/s, 8 objects/s recovering 2024-09-14T04:14:52.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:52.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:52.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:53.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:53.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:53.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:53.256 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:14:53.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:53.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:53.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:53.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:14:53.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:53.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:53.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:53.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:53.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:53.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:53 smithi001 ceph-mon[112165]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3937/216846 objects degraded (1.816%); 3858/216846 objects misplaced (1.779%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:14:53.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:53 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/216846 objects degraded (1.816%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:53.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:53 smithi001 ceph-mon[107399]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3937/216846 objects degraded (1.816%); 3858/216846 objects misplaced (1.779%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:14:53.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:53 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/216846 objects degraded (1.816%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:53.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:14:53.775 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 74985 74969 108.937 69.375 0.00990072 0.00917408 2024-09-14T04:14:53.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:14:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:53 smithi012 ceph-mon[101180]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3937/216846 objects degraded (1.816%); 3858/216846 objects misplaced (1.779%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:14:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:53 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/216846 objects degraded (1.816%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:53.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:53.944 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:53.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:54.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:54.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:54.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:54.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:54.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:54.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:54.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:54.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:54.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:54.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:54.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:54.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:54.777 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 76174 76158 108.146 74.3125 0.0236321 0.00924044 2024-09-14T04:14:54.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:54.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:54.913 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:54.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:55.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:55.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:55.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:55.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:55.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:55.332 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:55.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:55.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:55.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:55.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:55.612 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:55.714 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:55.752 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:55.777 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 77336 77320 107.357 72.625 0.00977542 0.00930906 2024-09-14T04:14:55.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:55.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:55.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:55.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:55.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:56.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:56.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:56.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:56.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:56.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:56.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:56.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:56.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:56.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:56.556 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:56 smithi001 ceph-mon[107399]: pgmap v897: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3937/219903 objects degraded (1.790%); 3858/219903 objects misplaced (1.754%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:14:56.556 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:56 smithi001 ceph-mon[112165]: pgmap v897: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3937/219903 objects degraded (1.790%); 3858/219903 objects misplaced (1.754%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:14:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:56 smithi012 ceph-mon[101180]: pgmap v897: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3937/219903 objects degraded (1.790%); 3858/219903 objects misplaced (1.754%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:14:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:56.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:56.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:56.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:56.777 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 78471 78455 106.565 70.9375 0.00791509 0.00937858 2024-09-14T04:14:56.820 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:14:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:14:56] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:14:56.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:56.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:56.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:56.932 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:14:56.932 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:14:56.932 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:14:56.932 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:14:56.945 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:14:56.945 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:14:56.992 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:14:56.992 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:14:56.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:57.028 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:14:57.028 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:14:57.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:57.057 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:14:57.058 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:14:57.090 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:14:57.090 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:14:57.123 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:14:57.124 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:14:57.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:57.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:57.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:57.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:57.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:57.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:57.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1043677779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:57.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1043677779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:57.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1043677779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:14:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:57.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:57.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:57.707 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:57.725 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.727 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:14:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:57.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 79516 79500 105.687 65.3125 0.0472288 0.00945642 2024-09-14T04:14:57.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:57.857 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:57.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:57.991 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:57.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:58.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:58.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:58.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:58.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:58.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:58.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:58.506 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:58 smithi001 ceph-mon[107399]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 3937/231798 objects degraded (1.698%); 3754/231798 objects misplaced (1.620%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:14:58.506 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:58 smithi001 ceph-mon[112165]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 3937/231798 objects degraded (1.698%); 3754/231798 objects misplaced (1.620%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:14:58.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:58.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:58 smithi012 ceph-mon[101180]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 3937/231798 objects degraded (1.698%); 3754/231798 objects misplaced (1.620%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:14:58.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:58.694 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:14:58.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 80651 80635 104.963 70.9375 0.0202924 0.00952216 2024-09-14T04:14:58.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:58.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:58.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:14:58.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:58.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:14:59.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:14:59.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:14:59.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:14:59.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:59.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:14:59.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:14:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:59.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:14:59.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:59.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:14:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/231798 objects degraded (1.698%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:59.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:14:59.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:14:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:14:59.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:14:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/231798 objects degraded (1.698%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:14:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/231798 objects degraded (1.698%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:14:59.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:14:59.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 81763 81747 104.238 69.5 0.0116218 0.00958708 2024-09-14T04:14:59.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:14:59.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:14:59.949 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:14:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:14:59.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:00.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:00.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:00.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:00.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:00.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:15:00.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:00.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:00.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:00 smithi012 ceph-mon[101180]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3937/234945 objects degraded (1.676%); 3754/234945 objects misplaced (1.598%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:15:00.617 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:00 smithi001 ceph-mon[112165]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3937/234945 objects degraded (1.676%); 3754/234945 objects misplaced (1.598%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:15:00.617 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:00 smithi001 ceph-mon[107399]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3937/234945 objects degraded (1.676%); 3754/234945 objects misplaced (1.598%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:15:00.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:00.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:00.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 82805 82789 103.456 65.125 0.0110012 0.00966127 2024-09-14T04:15:00.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:00.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:00.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:00.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:01.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:01.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:01.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:01.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:01.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:01.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:01.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:01.524 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:01.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:01.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:01.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:01.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 83845 83829 102.702 65 0.0109669 0.00973148 2024-09-14T04:15:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:01.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:01.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:01.954 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:01.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:02.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:02.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:02.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:02.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:02.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:02.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:02 smithi001 ceph-mon[107399]: pgmap v900: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 3937/241869 objects degraded (1.628%); 3754/241869 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:15:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:02 smithi001 ceph-mon[112165]: pgmap v900: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 3937/241869 objects degraded (1.628%); 3754/241869 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:15:02.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:02.502 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:02 smithi012 ceph-mon[101180]: pgmap v900: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 3937/241869 objects degraded (1.628%); 3754/241869 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:15:02.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:02.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:02.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:02.771 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:02.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 84958 84942 102.065 69.5625 0.0216489 0.00979193 2024-09-14T04:15:02.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:02.907 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:02.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:02.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:03.008 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:03.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:03.147 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:03.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:03.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:03.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:03.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:03.456 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:03.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:03.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:03.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:03.740 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:03.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 86058 86042 101.436 68.75 0.0113253 0.00985301 2024-09-14T04:15:03.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:03.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:03.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:03.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:03.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:04.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:04.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:04.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.157 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:04.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:04.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:04.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:04.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:04.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:04.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:04.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:04.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:04 smithi012 ceph-mon[101180]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3937/250035 objects degraded (1.575%); 3636/250035 objects misplaced (1.454%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:15:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/250035 objects degraded (1.575%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:04.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:04.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:04.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:04 smithi001 ceph-mon[112165]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3937/250035 objects degraded (1.575%); 3636/250035 objects misplaced (1.454%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:15:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/250035 objects degraded (1.575%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:04 smithi001 ceph-mon[107399]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3937/250035 objects degraded (1.575%); 3636/250035 objects misplaced (1.454%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:15:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/250035 objects degraded (1.575%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:04.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 87128 87112 100.796 66.875 0.00771419 0.00991576 2024-09-14T04:15:04.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:04.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:04.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:04.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:04.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:05.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:05.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:05.178 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:15:05.178 INFO:teuthology.orchestra.run.smithi001.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:15:05.178 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:15:05.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:05.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:05.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:05.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:05.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:05.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:05.639 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:05.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.680 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:05.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:05.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 88200 88184 100.182 67 0.00512925 0.00997609 2024-09-14T04:15:05.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:05.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:05.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:05.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:05.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:06.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:06.094 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:06.105 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:06.105 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:06.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[107399]: pgmap v902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3937/253113 objects degraded (1.555%); 3636/253113 objects misplaced (1.437%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:05 smithi001 ceph-mon[112165]: pgmap v902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3937/253113 objects degraded (1.555%); 3636/253113 objects misplaced (1.437%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:15:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:06.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:05 smithi012 ceph-mon[101180]: pgmap v902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 3937/253113 objects degraded (1.555%); 3636/253113 objects misplaced (1.437%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:15:06.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:06.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:06.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.534 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:06.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:06.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:06.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:06.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:06.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 15 89135 89120 99.4375 58.5 0.0315401 0.0100515 2024-09-14T04:15:06.812 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:06.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:06.941 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:15:06.942 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:06] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-09-14T04:15:06.945 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:06.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:07.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:07.070 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/366238903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.070 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:06 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.085 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:07.187 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/366238903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:06 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/366238903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:06 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:07.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:07.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:07.367 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:07.422 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:07.469 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.517 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:07.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:07.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:07.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:07.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:07.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:07.778 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 90162 90146 98.818 64.125 0.0165531 0.0101143 2024-09-14T04:15:07.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:07.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:07.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:15:07.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:07.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:08.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:08.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:08.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:08.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[112165]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[112165]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/264084 objects degraded (1.491%); 3523/264084 objects misplaced (1.334%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:08.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[107399]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T04:15:08.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:07 smithi001 ceph-mon[107399]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/264084 objects degraded (1.491%); 3523/264084 objects misplaced (1.334%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:08.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:08.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:07 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:08.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:07 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:08.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:07 smithi012 ceph-mon[101180]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T04:15:08.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:07 smithi012 ceph-mon[101180]: pgmap v904: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/264084 objects degraded (1.491%); 3523/264084 objects misplaced (1.334%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:08.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:08.335 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:08.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:08.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:08.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:08.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:08.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:08.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:08.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:08.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 91176 91160 98.2062 63.375 0.0108432 0.0101767 2024-09-14T04:15:08.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:08.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:08.903 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:08.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:08.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:09.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:09.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:09.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:09.212 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:09.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:09.355 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/264084 objects degraded (1.491%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:09.355 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/264084 objects degraded (1.491%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.368 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:09.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:09.469 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:09.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/264084 objects degraded (1.491%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:09.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:09.640 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:09.742 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:09.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 92210 92194 97.637 64.625 0.00972431 0.0102367 2024-09-14T04:15:09.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:09.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:09.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:09.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:09.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:10.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:10.058 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:10.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:10.160 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:10.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:10.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:10 smithi001 ceph-mon[107399]: pgmap v905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/267009 objects degraded (1.474%); 3523/267009 objects misplaced (1.319%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:10.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:10 smithi001 ceph-mon[112165]: pgmap v905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/267009 objects degraded (1.474%); 3523/267009 objects misplaced (1.319%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:10.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:10.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:10.359 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:10 smithi012 ceph-mon[101180]: pgmap v905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 3937/267009 objects degraded (1.474%); 3523/267009 objects misplaced (1.319%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:10.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:10.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:10.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:10.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:10.709 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:10.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:10.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:15:10.779412+0000 min lat: 0.00131144 max lat: 2.13766 avg lat: 0.0102959 2024-09-14T04:15:10.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:15:10.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 93226 93210 97.0681 63.5 0.0515306 0.0102959 2024-09-14T04:15:10.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:10.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:10.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:10.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:11.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:11.063 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:11.063 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:11.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:11.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:11.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:11.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.310 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:11.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:11.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:11.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:11.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:11.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:11.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:11.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:11.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 94127 94111 96.4 56.3125 0.00747689 0.0103673 2024-09-14T04:15:11.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:11.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:11.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:11.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:15:11.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:15:11.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:12.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:12.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:12.115 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:15:12.115 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:15:12.147 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:12.168 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:15:12.169 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:15:12.186 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:12.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:12.218 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:15:12.218 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:15:12.252 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:15:12.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:15:12.281 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:15:12.282 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:15:12.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:12.315 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:15:12.315 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:15:12.356 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:15:12.356 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:15:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:12.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:12.365 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:12.391 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:15:12.391 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:15:12.393 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:12 smithi001 ceph-mon[107399]: pgmap v906: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3937/273789 objects degraded (1.438%); 3523/273789 objects misplaced (1.287%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:12.393 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2370823762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:12.393 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:12 smithi001 ceph-mon[112165]: pgmap v906: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3937/273789 objects degraded (1.438%); 3523/273789 objects misplaced (1.287%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:12.393 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2370823762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:12.459 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:15:12.459 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:15:12.459 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-14T04:15:12.459 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:15:12.459 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:15:12.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-14T04:15:12.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:12.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:12 smithi012 ceph-mon[101180]: pgmap v906: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3937/273789 objects degraded (1.438%); 3523/273789 objects misplaced (1.287%); 1.4 MiB/s, 23 objects/s recovering 2024-09-14T04:15:12.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2370823762' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:12.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:12.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:12.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:12.779 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 94885 94869 95.6093 47.375 0.015995 0.0104534 2024-09-14T04:15:12.810 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:12.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:12.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:12.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:13.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:13.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:13.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:13.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:13.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:13.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:13.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:13.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:13.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:13.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:13.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:13.653 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:13.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:13.777 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:13.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 95863 95847 95.0619 61.125 0.00683294 0.0105143 2024-09-14T04:15:13.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:13.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:13.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:13.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:13.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:14.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:14.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:14.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:14.199 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:14.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:14.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:14.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:14 smithi001 ceph-mon[107399]: pgmap v907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 3937/281013 objects degraded (1.401%); 3393/281013 objects misplaced (1.207%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/281013 objects degraded (1.401%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:14 smithi001 ceph-mon[112165]: pgmap v907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 3937/281013 objects degraded (1.401%); 3393/281013 objects misplaced (1.207%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/281013 objects degraded (1.401%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:14.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:14 smithi012 ceph-mon[101180]: pgmap v907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 3937/281013 objects degraded (1.401%); 3393/281013 objects misplaced (1.207%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:14.477 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/281013 objects degraded (1.401%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:14.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:14.525 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:14.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:14.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:14.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:14.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 96839 96823 94.5295 61 0.011512 0.0105725 2024-09-14T04:15:14.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:14.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:14.929 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:14.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:15.030 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:15.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:15.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:15.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:15.250 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:15:15.250 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:15:15.250 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:15:15.250 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:15:15.250 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:15:15.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.366 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:15.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:15.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:15.504 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:15.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:15.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:15.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:15:15.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:15.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 97753 97737 93.954 57.125 0.00688618 0.0106377 2024-09-14T04:15:15.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:15.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:15.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:15.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:15.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:16.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:16.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:16.091 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:16.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:16.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:16.321 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:15:16.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:16.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:16.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:16 smithi012 ceph-mon[101180]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 3937/283635 objects degraded (1.388%); 3393/283635 objects misplaced (1.196%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/816567700' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:16 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:16.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:16.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[112165]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 3937/283635 objects degraded (1.388%); 3393/283635 objects misplaced (1.196%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:16.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/816567700' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[107399]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 3937/283635 objects degraded (1.388%); 3393/283635 objects misplaced (1.196%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/816567700' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:16.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:16.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 98604 98588 93.3364 53.1875 0.00829238 0.0107085 2024-09-14T04:15:16.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:16.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:16.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:16.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:16.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:16] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-09-14T04:15:17.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:17.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:17.151 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:17.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:17.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:17.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:17.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[107399]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:17.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:17 smithi001 ceph-mon[112165]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T04:15:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:17.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:17.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:17 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:17.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:17 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:17.587 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:17 smithi012 ceph-mon[101180]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T04:15:17.613 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:17.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:17.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:17.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 99601 99585 92.8733 62.3125 0.00794409 0.0107626 2024-09-14T04:15:17.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:15:17.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:17.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:17.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:17.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:18.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:18.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:18.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:18.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:18.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:18.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.347 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:18.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:18.409 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:15:18.409 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2024-09-14T04:15:18.409 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:15:18.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:18 smithi001 ceph-mon[107399]: pgmap v910: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 3937/292932 objects degraded (1.344%); 3278/292932 objects misplaced (1.119%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:18 smithi001 ceph-mon[112165]: pgmap v910: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 3937/292932 objects degraded (1.344%); 3278/292932 objects misplaced (1.119%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:18.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:18.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:18.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:18.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:18 smithi012 ceph-mon[101180]: pgmap v910: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 3937/292932 objects degraded (1.344%); 3278/292932 objects misplaced (1.119%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:18.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:18.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:18.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:18.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:18.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:18.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 100457 100441 92.2922 53.5 0.0117086 0.0108289 2024-09-14T04:15:18.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:18.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:18.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:19.008 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:19.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:19.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:19.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:19.184 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:19.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:19.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:19.409 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/292932 objects degraded (1.344%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:19.410 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/292932 objects degraded (1.344%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:19.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:19.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/292932 objects degraded (1.344%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:19.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:19.638 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:19.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:19.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:19.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 101365 101349 91.7771 56.75 0.0160311 0.0108905 2024-09-14T04:15:19.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:19.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:19.913 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:15:19.913 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:15:19.955 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:15:19.956 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:15:19.989 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:15:19.989 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:15:19.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:20.025 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:15:20.025 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:15:20.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:20.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:20.057 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:15:20.057 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:15:20.092 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:15:20.092 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:15:20.128 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:15:20.128 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:15:20.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:20.162 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:15:20.162 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:15:20.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:20.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:20.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:20.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:20.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:20.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:20.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:20.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:20.605 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[112165]: pgmap v911: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 3937/295626 objects degraded (1.332%); 3278/295626 objects misplaced (1.109%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:20.605 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1491766922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:20.605 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:20.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:20.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[107399]: pgmap v911: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 3937/295626 objects degraded (1.332%); 3278/295626 objects misplaced (1.109%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:20.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1491766922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:20.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:20.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:20.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:20.691 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:15:20.714 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:20.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 102220 102204 91.2286 53.4375 0.0323809 0.0109563 2024-09-14T04:15:20.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:20 smithi012 ceph-mon[101180]: pgmap v911: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 3937/295626 objects degraded (1.332%); 3278/295626 objects misplaced (1.109%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1491766922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:20.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:20.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:20.950 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:20.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:20.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:21.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:21.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:21.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:21.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:21.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:21.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:21.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:21.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:21.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:21.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:21.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:21.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:21.737 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:21.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 103079 103063 90.696 53.6875 0.020323 0.0110196 2024-09-14T04:15:21.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:21.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:21.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:21.972 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:21.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:22.074 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:22.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:22.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:22.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.257 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:22.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:22.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:22.439 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:22 smithi012 ceph-mon[101180]: pgmap v912: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 3937/301365 objects degraded (1.306%); 3278/301365 objects misplaced (1.088%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:22.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:22.513 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:22.513 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:22.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:22.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:22.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:22.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:22 smithi001 ceph-mon[107399]: pgmap v912: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 3937/301365 objects degraded (1.306%); 3278/301365 objects misplaced (1.088%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:22 smithi001 ceph-mon[112165]: pgmap v912: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 3937/301365 objects degraded (1.306%); 3278/301365 objects misplaced (1.088%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:22.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:22.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 103974 103958 90.2132 55.9375 0.0127965 0.0110792 2024-09-14T04:15:22.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:22.907 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:22.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:22.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:23.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:23.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:23.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:15:23.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:15:23.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:23.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:23.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:23.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:23.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:15:23.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:23.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:23.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:23.737 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:23.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:23.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:15:23.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 104574 104558 89.4866 37.5 0.0350854 0.0111691 2024-09-14T04:15:23.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:23.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:23.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:23.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:24.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:24.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:24.110 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:15:24.110 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:15:24.110 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-14T04:15:24.110 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-14T04:15:24.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2 2024-09-14T04:15:24.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:24.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:24.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:24.360 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:24 smithi001 systemd[1]: Stopping Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:15:24.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:24.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:24.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:24.498 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:24 smithi012 ceph-mon[101180]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 3937/308103 objects degraded (1.278%); 3151/308103 objects misplaced (1.023%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/308103 objects degraded (1.278%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:24.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:24.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:24 smithi001 ceph-mon[112165]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 3937/308103 objects degraded (1.278%); 3151/308103 objects misplaced (1.023%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/308103 objects degraded (1.278%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:24 smithi001 ceph-mon[107399]: pgmap v913: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 3937/308103 objects degraded (1.278%); 3151/308103 objects misplaced (1.023%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/308103 objects degraded (1.278%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:24.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:24.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:24.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 105450 105434 89.0172 54.75 0.00918803 0.0112286 2024-09-14T04:15:24.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:24.896 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:24.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:24.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:25.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:25.051 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:25.153 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:25.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:25.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:25.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:25.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:25.483 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:25.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:25.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:25.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:25.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:25.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 106374 106358 88.6004 57.75 0.00799028 0.0112814 2024-09-14T04:15:25.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:25.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:25.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:25.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:26.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:26.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:26.160 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:26.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:26.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:26.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:26.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:26.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:26.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:26.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.576 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:26.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:26.677 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:26.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:26 smithi001 ceph-mon[107399]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3937/310419 objects degraded (1.268%); 3151/310419 objects misplaced (1.015%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:26 smithi001 ceph-mon[112165]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3937/310419 objects degraded (1.268%); 3151/310419 objects misplaced (1.015%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:26.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:26.735 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:26 smithi012 ceph-mon[101180]: pgmap v914: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s; 3937/310419 objects degraded (1.268%); 3151/310419 objects misplaced (1.015%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:15:26.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 107242 107226 88.1484 54.25 0.00640025 0.0113392 2024-09-14T04:15:26.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:26.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:26.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:26.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:26] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-14T04:15:27.008 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:27.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:27.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:27.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:27.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:27.377 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:27.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:27.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:27.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:27.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:27.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:27.722 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:27.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 108094 108078 87.6953 53.25 0.00392956 0.0113885 2024-09-14T04:15:27.824 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:27.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:27.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:27.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:28.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:28.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:28.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:28.246 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:28.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:28.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:28.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:28.419 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:28.419 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:28.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:28.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:28.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:28.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:28.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:28 smithi001 ceph-mon[107399]: pgmap v915: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 3937/319062 objects degraded (1.234%); 2973/319062 objects misplaced (0.932%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:15:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:28 smithi001 ceph-mon[112165]: pgmap v915: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 3937/319062 objects degraded (1.234%); 2973/319062 objects misplaced (0.932%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:15:28.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:28.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 108651 108635 87.0174 34.8125 0.0114487 0.0114844 2024-09-14T04:15:28.792 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:28 smithi012 ceph-mon[101180]: pgmap v915: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 3937/319062 objects degraded (1.234%); 2973/319062 objects misplaced (0.932%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:15:28.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:28.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:28.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:29.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:29.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:29.172 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:29.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:29.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:29.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:29.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:29.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:29.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:15:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:29.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:29.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:29.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:29.654 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/319062 objects degraded (1.234%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/319062 objects degraded (1.234%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/319062 objects degraded (1.234%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:29.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:29.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:29.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 109474 109458 86.5666 51.4375 0.0125506 0.0115464 2024-09-14T04:15:29.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:29.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:29.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:30.019 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:30.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:30.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:30.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:30.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:30.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:30.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:30.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.351 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:30.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:30.377 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:15:30.378 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:15:30.400 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:15:30.426 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:15:30.426 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:15:30.452 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:30.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:15:30.473 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:15:30.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:30.510 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:15:30.510 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:15:30.547 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:15:30.548 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:15:30.584 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:15:30.584 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:15:30.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:30.620 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:15:30.620 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:15:30.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:30.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[112165]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3937/321411 objects degraded (1.225%); 2973/321411 objects misplaced (0.925%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:15:30.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3897407534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.655 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/186006266' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:30.655 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:15:30.656 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:15:30.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[107399]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3937/321411 objects degraded (1.225%); 2973/321411 objects misplaced (0.925%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:15:30.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3897407534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/186006266' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:30.663 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:30 smithi012 ceph-mon[101180]: pgmap v916: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3937/321411 objects degraded (1.225%); 2973/321411 objects misplaced (0.925%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:15:30.664 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3897407534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.664 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:30 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:15:30.664 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/186006266' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:30.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:30.771 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:30.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:15:30.790927+0000 min lat: 0.00131144 max lat: 2.13766 avg lat: 0.0116051 2024-09-14T04:15:30.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:15:30.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 110298 110282 86.1283 51.5 0.0149045 0.0116051 2024-09-14T04:15:30.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:30.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:30.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:31.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:31.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:31.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:31.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:31.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:31.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:31.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:31.466 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:31.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:31 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:31 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:15:31.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:31 smithi012 ceph-mon[101180]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T04:15:31.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:31.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:31.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:15:31.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[112165]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T04:15:31.674 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:31.674 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:15:31.674 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-mon[107399]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T04:15:31.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.760 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:31.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:31.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 111201 111185 85.7618 56.4375 0.0151446 0.0116551 2024-09-14T04:15:31.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:31.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:31.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:31.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:15:31.803+0000 7f4ecc171640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:15:31.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:15:31.803+0000 7f4ecc171640 -1 osd.2 259 *** Got signal Terminated *** 2024-09-14T04:15:31.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[136910]: 2024-09-14T04:15:31.803+0000 7f4ecc171640 -1 osd.2 259 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:15:32.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:32.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:32.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:32.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:32.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:32.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:32.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:32.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:32.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:32.580 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:32 smithi012 ceph-mon[101180]: pgmap v918: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3937/326511 objects degraded (1.206%); 2973/326511 objects misplaced (0.911%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:15:32.581 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:32 smithi012 ceph-mon[101180]: osd.2 marked itself down and dead 2024-09-14T04:15:32.587 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:32.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:32.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:32.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:32 smithi001 ceph-mon[112165]: pgmap v918: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3937/326511 objects degraded (1.206%); 2973/326511 objects misplaced (0.911%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:15:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:32 smithi001 ceph-mon[112165]: osd.2 marked itself down and dead 2024-09-14T04:15:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:32 smithi001 ceph-mon[107399]: pgmap v918: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3937/326511 objects degraded (1.206%); 2973/326511 objects misplaced (0.911%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:15:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:32 smithi001 ceph-mon[107399]: osd.2 marked itself down and dead 2024-09-14T04:15:32.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 111770 111754 85.1497 35.5625 0.00472296 0.0116882 2024-09-14T04:15:32.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:32.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:32.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:32.875 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:15:32.876 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:15:32.876 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:15:32.876 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:15:32.876 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:15:32.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:32.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:32.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:33.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:33.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:33.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:33.217 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:33.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:33.343 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:33.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:33.476 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:33.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:15:33.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:33.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:33.672 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:15:33.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:15:33.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[112165]: osdmap e260: 8 total, 7 up, 8 in 2024-09-14T04:15:33.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[112165]: osdmap e261: 8 total, 7 up, 8 in 2024-09-14T04:15:33.716 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:15:33.716 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[107399]: osdmap e260: 8 total, 7 up, 8 in 2024-09-14T04:15:33.716 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:33 smithi001 ceph-mon[107399]: osdmap e261: 8 total, 7 up, 8 in 2024-09-14T04:15:33.716 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:33 smithi001 podman[236824]: 2024-09-14 04:15:33.446431392 +0000 UTC m=+4.092430846 container died 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:15:33.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:33.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:33.791 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 112076 112060 84.3544 19.125 0.0074281 0.0118491 2024-09-14T04:15:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:33 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:15:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:33 smithi012 ceph-mon[101180]: osdmap e260: 8 total, 7 up, 8 in 2024-09-14T04:15:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:33 smithi012 ceph-mon[101180]: osdmap e261: 8 total, 7 up, 8 in 2024-09-14T04:15:33.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:33.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:33.986 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:34.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:34.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:34.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:34.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:34.227 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:34.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:34.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:34.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:15:34.405 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:34 smithi001 podman[236824]: 2024-09-14 04:15:34.211202317 +0000 UTC m=+4.857201766 container cleanup 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:15:34.405 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:34 smithi001 bash[236824]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2 2024-09-14T04:15:34.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:34.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:34.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:34.709 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[107399]: pgmap v920: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 stale+active+clean, 90 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 3937/330978 objects degraded (1.190%); 2799/330978 objects misplaced (0.846%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:15:34.709 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3937/330978 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[112165]: pgmap v920: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 stale+active+clean, 90 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 3937/330978 objects degraded (1.190%); 2799/330978 objects misplaced (0.846%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.710 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3937/330978 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:34.710 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:34 smithi001 podman[237269]: 2024-09-14 04:15:34.404777674 +0000 UTC m=+1.283311888 container remove 35b8e87dea9d847e6007a949dba7138f752d17d6fb58aaa3f65abbdecb2f7f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-14T04:15:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:34.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:34.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 112922 112906 83.9791 52.875 0.0252429 0.0119023 2024-09-14T04:15:34.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:34 smithi012 ceph-mon[101180]: pgmap v920: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 stale+active+clean, 90 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 3937/330978 objects degraded (1.190%); 2799/330978 objects misplaced (0.846%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:15:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3937/330978 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:15:34.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:34.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:34.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:34.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:34.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:35.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:35.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:35.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:35.233 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:35.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:35.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:35.273 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:35.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:35.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:35.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:35.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:35.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:35.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:35.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:35.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 113845 113829 83.6698 57.6875 0.0145821 0.0119469 2024-09-14T04:15:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:35.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:35.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: pgmap v922: 99 pgs: 5 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 35 MiB/s wr, 567 op/s; 11000/331629 objects degraded (3.317%); 2799/331629 objects misplaced (0.844%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:35.948 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:35.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:35.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.44714.0:166779 9.d e153/135 9:b228385c:::benchmark_data_smithi001_156540_object35185:head v 153'10400, mlcod=153'10398)", 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 545.81025631299997, 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18896571700000001, 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.010 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193930+0000", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193929+0000", 2024-09-14T04:15:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193937+0000", 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2699999999999999e-06 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193947+0000", 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0465000000000001e-05 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.193953+0000", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3790000000000003e-06 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195833+0000", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001880044 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.195851+0000", 2024-09-14T04:15:36.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7334999999999999e-05 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.230956+0000", 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.035105619999999997 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382896+0000", 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.151939308 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.014 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:166789 9.5 9:a97ca575:::benchmark_data_smithi001_156540_object35195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 545.80313023400004, 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18248584100000001, 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166789 2024-09-14T04:15:36.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201056+0000", 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201055+0000", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201059+0000", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.873e-06 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201072+0000", 2024-09-14T04:15:36.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2414e-05 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.201078+0000", 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3459999999999999e-06 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211075+0000", 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0099971409999999993 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211180+0000", 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010500799999999999 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 7", 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.211291+0000", 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111189 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.382943+0000", 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17165192400000001 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383362+0000", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041839200000000002 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383402+0000", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0034999999999999e-05 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:30.383542+0000", 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000140316 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.020 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178516 9.7 9:e7cca128:::benchmark_data_smithi001_156540_object46922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.69126920199994, 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0890654740000001, 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178516 2024-09-14T04:15:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312917+0000", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312919+0000", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7429999999999998e-06 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312926+0000", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6290000000000004e-06 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.312931+0000", 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.13e-06 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313179+0000", 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024747799999999998 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313398+0000", 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219287 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.313540+0000", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014250299999999999 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.318169+0000", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046281289999999999 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322748+0000", 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004579751 2024-09-14T04:15:36.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348314+0000", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025566169 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.348324+0000", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0469999999999999e-06 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401919+0000", 2024-09-14T04:15:36.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.053594996 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.401983+0000", 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4016999999999994e-05 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178520 9.8 9:1c2e04c6:::benchmark_data_smithi001_156540_object46926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.68436740599998, 2024-09-14T04:15:36.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0110979040000001, 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178520 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:15:36.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319819+0000", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319821+0000", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1720000000000001e-06 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319826+0000", 2024-09-14T04:15:36.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7339999999999996e-06 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319831+0000", 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2799999999999997e-06 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320563+0000", 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073229899999999995 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320671+0000", 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107984 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.320794+0000", 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122669 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.332330+0000", 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011536103000000001 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343578+0000", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011247823000000001 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.343588+0000", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9939999999999995e-06 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330904+0000", 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98731630800000003 2024-09-14T04:15:36.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330917+0000", 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2883e-05 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178521 9.7 9:e7156783:::benchmark_data_smithi001_156540_object46927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.68434740199996, 2024-09-14T04:15:36.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069582149999999, 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178521 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319839+0000", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319838+0000", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319841+0000", 2024-09-14T04:15:36.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.086e-06 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319846+0000", 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6110000000000001e-06 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.319850+0000", 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.211e-06 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322773+0000", 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029226109999999999 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.322906+0000", 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013334800000000001 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.323044+0000", 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137897 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350098+0000", 2024-09-14T04:15:36.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027053395000000001 2024-09-14T04:15:36.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350106+0000", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5750000000000001e-06 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416318+0000", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.066211786 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416396+0000", 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8298000000000001e-05 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426758+0000", 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010361354 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426797+0000", 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9487999999999999e-05 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.041 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178528 9.f 9:f3e5f4f3:::benchmark_data_smithi001_156540_object46934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.675840902, 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.087887475, 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178528 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328345+0000", 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328344+0000", 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328349+0000", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0950000000000003e-06 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328361+0000", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1915e-05 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.328367+0000", 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6549999999999998e-06 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350130+0000", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021762786999999999 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350137+0000", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8630000000000002e-06 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368572+0000", 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.018434893 2024-09-14T04:15:36.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368729+0000", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015681500000000001 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.368871+0000", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141779 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402186+0000", 2024-09-14T04:15:36.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033314990000000003 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.402204+0000", 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8601e-05 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416128+0000", 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013924158000000001 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416185+0000", 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6869e-05 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416233+0000", 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7602999999999998e-05 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178532 9.7 9:e0e67f4c:::benchmark_data_smithi001_156540_object46938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-14T04:15:36.048 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.668896797, 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.088792084, 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178532 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335289+0000", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335288+0000", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335293+0000", 2024-09-14T04:15:36.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2470000000000002e-06 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335303+0000", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0829000000000001e-05 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.335310+0000", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9009999999999997e-06 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350158+0000", 2024-09-14T04:15:36.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014848175999999999 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.350165+0000", 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416444+0000", 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0662793960000001 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416514+0000", 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9778999999999997e-05 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.416588+0000", 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3963999999999999e-05 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423845+0000", 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072567669999999999 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423979+0000", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013418300000000001 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.423990+0000", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1459e-05 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424029+0000", 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.892e-05 2024-09-14T04:15:36.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.424082+0000", 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2386000000000003e-05 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178537 9.8 9.6a0d3bf8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.65311396200002, 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97989574099999999, 2024-09-14T04:15:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178537 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351072+0000", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351075+0000", 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.666e-06 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351081+0000", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4840000000000001e-06 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351086+0000", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0699999999999997e-06 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351541+0000", 2024-09-14T04:15:36.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045494699999999998 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.351549+0000", 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.498e-06 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330958+0000", 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97940985199999997 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.330968+0000", 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6840000000000008e-06 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178547 9.8 9.c42478a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.62510766499997, 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.952303813, 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178547 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379079+0000", 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379078+0000", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379083+0000", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0189999999999997e-06 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379094+0000", 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1126e-05 2024-09-14T04:15:36.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379099+0000", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379137+0000", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.786e-05 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0891e-05 2024-09-14T04:15:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331371+0000", 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95222277899999996 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.331382+0000", 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1686e-05 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:178548 9.f 9:f6c46334:::benchmark_data_smithi001_156540_object46954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 530.62503771900003, 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0510430099999999, 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178548 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379149+0000", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379148+0000", 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379152+0000", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0289999999999999e-06 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379161+0000", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8910000000000001e-06 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379166+0000", 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1440000000000002e-06 2024-09-14T04:15:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379197+0000", 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0839000000000001e-05 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:45.379202+0000", 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1309999999999997e-06 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420298+0000", 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410965350000001 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420384+0000", 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5554000000000001e-05 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.420466+0000", 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2263999999999997e-05 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.426757+0000", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006290459 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430060+0000", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033035619999999999 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430121+0000", 2024-09-14T04:15:36.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1101999999999999e-05 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430165+0000", 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4026000000000002e-05 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:06:46.430192+0000", 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6142000000000001e-05 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262060 9.5 9:aa922bae:::benchmark_data_smithi001_156540_object130466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.52095118599999, 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.199390137, 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262060 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483235+0000", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483234+0000", 2024-09-14T04:15:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483238+0000", 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483247+0000", 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8869999999999998e-06 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483252+0000", 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.882e-06 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483300+0000", 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7879e-05 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.483306+0000", 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7139999999999998e-06 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647214+0000", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16390846000000001 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647330+0000", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011582 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.647443+0000", 2024-09-14T04:15:36.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011323999999999999 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653349+0000", 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059055039999999998 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.653931+0000", 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058244600000000003 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682380+0000", 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028448703999999998 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682433+0000", 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3291000000000002e-05 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.682625+0000", 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019192999999999999 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.44714.0:262062 9.b 9:db855ab8:::benchmark_data_smithi001_156540_object130468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.51105207900002, 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26506028100000001, 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.44714", 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2907999036", 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 262062 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493134+0000", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493133+0000", 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:36.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: pgmap v922: 99 pgs: 5 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 35 MiB/s wr, 567 op/s; 11000/331629 objects degraded (3.317%); 2799/331629 objects misplaced (0.844%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:36.082 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: pgmap v922: 99 pgs: 5 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 stale+active+clean, 3 active+undersized+degraded, 84 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 35 MiB/s wr, 567 op/s; 11000/331629 objects degraded (3.317%); 2799/331629 objects misplaced (0.844%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:36.083 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493140+0000", 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7069999999999992e-06 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493155+0000", 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4440999999999999e-05 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493162+0000", 2024-09-14T04:15:36.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6499999999999999e-06 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493204+0000", 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2209000000000002e-05 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.493212+0000", 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9929999999999995e-06 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654542+0000", 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.161330167 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654750+0000", 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020799300000000001 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.654877+0000", 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012702300000000001 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663336+0000", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084589330000000001 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.663377+0000", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1032999999999998e-05 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757657+0000", 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094279643999999996 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.757711+0000", 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4691000000000002e-05 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:08:16.758194+0000", 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048327800000000002 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.088 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85190 10.9 10.cff927e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 346.74655481600001, 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.954429588, 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85190 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257631+0000", 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257630+0000", 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257695+0000", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4649999999999999e-05 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257714+0000", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8902999999999998e-05 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257722+0000", 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7770000000000002e-06 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257768+0000", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6112999999999997e-05 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257776+0000", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.191e-06 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212048+0000", 2024-09-14T04:15:36.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95427256999999999 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212061+0000", 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2573e-05 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85203 10.9 10.6280d1d9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 346.73356096999998, 2024-09-14T04:15:36.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94195282000000002, 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85203 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270625+0000", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270624+0000", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270697+0000", 2024-09-14T04:15:36.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2700999999999994e-05 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270712+0000", 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4982e-05 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270719+0000", 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7500000000000003e-06 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270772+0000", 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3193999999999999e-05 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270780+0000", 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4719999999999998e-06 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212569+0000", 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94178903899999999 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212578+0000", 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1409999999999994e-06 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85221 10.9 10.af3d50f9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 346.715714155, 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92413678799999999, 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.098 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85221 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288472+0000", 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288471+0000", 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288525+0000", 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3662999999999998e-05 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288537+0000", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2326e-05 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288543+0000", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5550000000000002e-06 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288594+0000", 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0821e-05 2024-09-14T04:15:36.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.288602+0000", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8759999999999994e-06 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212602+0000", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92399942800000001 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212609+0000", 2024-09-14T04:15:36.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85230 10.9 10.520f1ec9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 346.69580456, 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90426250600000002, 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:15:36.103 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85230 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308382+0000", 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308380+0000", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308415+0000", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4962999999999997e-05 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308432+0000", 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7719000000000001e-05 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308439+0000", 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9399999999999996e-06 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308497+0000", 2024-09-14T04:15:36.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7686000000000003e-05 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.308508+0000", 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0671e-05 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212636+0000", 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90412800299999996 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.212644+0000", 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5329999999999994e-06 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:189946 10.3 e230/184 10:cab92b5d:::benchmark_data_smithi001_190881_object74806:head v 230'11816, mlcod=230'11814)", 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.167749+0000", 2024-09-14T04:15:36.108 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 205.836437241, 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35705347700000001, 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167749+0000", 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167749+0000", 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167748+0000", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167755+0000", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5989999999999998e-06 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167765+0000", 2024-09-14T04:15:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0448999999999999e-05 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167772+0000", 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1580000000000002e-06 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167817+0000", 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5921000000000003e-05 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.167839+0000", 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1274e-05 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.172201+0000", 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043618859999999997 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524803+0000", 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35260193000000001 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.112 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:189938 10.7 e230/202 10:e77d48d0:::benchmark_data_smithi001_190881_object74798:head v 230'12051, mlcod=0'0)", 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.169987+0000", 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 205.834198856, 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35482930299999998, 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.169987+0000", 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.169987+0000", 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.169986+0000", 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.169995+0000", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6000000000000007e-06 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.170009+0000", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4085e-05 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.170017+0000", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6159999999999996e-06 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.170090+0000", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2776999999999998e-05 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.170115+0000", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5556999999999999e-05 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524780+0000", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35466486200000003 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524817+0000", 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6779999999999997e-05 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:189943 10.7 e230/202 10:e71d0422:::benchmark_data_smithi001_190881_object74803:head v 230'12052, mlcod=0'0)", 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.172647+0000", 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 205.83153884800001, 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38042817099999998, 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.172647+0000", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.172647+0000", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.172647+0000", 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:15:36.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.172653+0000", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8329999999999996e-06 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524779+0000", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35212577099999998 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524789+0000", 2024-09-14T04:15:36.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3100000000000006e-06 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524848+0000", 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9249000000000001e-05 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524876+0000", 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7937999999999999e-05 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.552919+0000", 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028043379 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.553076+0000", 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000156539 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:189950 10.6 10:6293d821:::benchmark_data_smithi001_190881_object74810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:15:36.121 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.174003+0000", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 205.83018296200001, 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37634535699999999, 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 189950 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.174003+0000", 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.174003+0000", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.174001+0000", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.174010+0000", 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.354e-06 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524761+0000", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35075160100000002 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524778+0000", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6543999999999999e-05 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524831+0000", 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2669e-05 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.524932+0000", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010172 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.525017+0000", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4554000000000004e-05 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.535235+0000", 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010218293 2024-09-14T04:15:36.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.546409+0000", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011173885 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.550252+0000", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003842888 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.550310+0000", 2024-09-14T04:15:36.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8291999999999997e-05 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.550349+0000", 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8442000000000002e-05 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:36.127 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:15:36.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:36.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.308 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:36.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:36.409 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:36.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:36.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:36 smithi001 podman[237521]: 2024-09-14 04:15:36.169842873 +0000 UTC m=+1.315842275 container died 9324edfdfc0df7a7f0bde3d73a1f97fe0d418f5321045ad8c7402dab17392e3a (image=quay.ceph.io/ceph-ci/ceph:reef, name=focused_mccarthy, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:15:36.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:36.590 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:36.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:36.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:36.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:36.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 114820 114804 83.4055 60.9375 0.00602013 0.0119844 2024-09-14T04:15:36.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:36.857 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:36.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:36.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:36.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:15:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:36] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:15:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:37.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:37.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:37.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:37.381 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:37.459 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.496 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:37.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:37.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.640 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:37.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:37.782 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:37.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 115797 115781 83.1487 61.0625 0.0314795 0.0120202 2024-09-14T04:15:37.884 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:37.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:37.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:38.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:38.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:38.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:38.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:38.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:38.334 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:38.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:38.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:38.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:38.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:38 smithi012 ceph-mon[101180]: pgmap v923: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 46490/342705 objects degraded (13.566%); 2522/342705 objects misplaced (0.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T04:15:38.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:38.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:38.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:38.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:38 smithi001 ceph-mon[107399]: pgmap v923: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 46490/342705 objects degraded (13.566%); 2522/342705 objects misplaced (0.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T04:15:38.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:38 smithi001 ceph-mon[112165]: pgmap v923: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 46490/342705 objects degraded (13.566%); 2522/342705 objects misplaced (0.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-14T04:15:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:38.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:38.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 116740 116724 82.8736 58.9375 0.0179301 0.0120615 2024-09-14T04:15:38.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:38.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:38.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:38.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:38.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:39.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:39.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:39.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:39.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:39.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:39.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:39.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:39.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:39.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 46490/342705 objects degraded (13.566%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:39.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:39.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 46490/342705 objects degraded (13.566%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:39.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 46490/342705 objects degraded (13.566%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:39.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 117458 117442 82.4467 44.875 0.00385366 0.0120966 2024-09-14T04:15:39.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:39.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:39.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:39.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:39.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:40.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:40.095 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:40.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:40.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:40.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:40.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:40.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:40 smithi001 ceph-mon[107399]: pgmap v924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 46777/343566 objects degraded (13.615%); 2522/343566 objects misplaced (0.734%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:15:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:40 smithi001 ceph-mon[112165]: pgmap v924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 46777/343566 objects degraded (13.615%); 2522/343566 objects misplaced (0.734%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:15:40.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:40.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:40 smithi012 ceph-mon[101180]: pgmap v924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 46777/343566 objects degraded (13.615%); 2522/343566 objects misplaced (0.734%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:15:40.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:40.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:40.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:40.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:40.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:40.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0119 2024-09-14T04:15:40.792 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 118312 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 82.1502 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 42.1707 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 237.125 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1314 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 674.731 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3794 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.012168 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.0292873 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 2.13766 2024-09-14T04:15:40.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00131144 2024-09-14T04:15:40.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:15:40.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:40.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:40.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:41.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:41.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:41.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:41.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:41.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:41.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:41.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:41.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:41.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:41.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:41.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:41.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:41.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:41.861 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:41.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:41.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:41.974 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:119490 11.f 11:f3a5a920:::benchmark_data_smithi001_228075_object1175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:41.946456+0000", 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.026004016000000001, 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026026909000000001, 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:15:41.975 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119490 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946456+0000", 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946456+0000", 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946455+0000", 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:41.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946461+0000", 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0000000000000002e-06 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946478+0000", 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6923e-05 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:41.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946485+0000", 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5660000000000001e-06 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946533+0000", 2024-09-14T04:15:41.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7426000000000003e-05 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946641+0000", 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010869 2024-09-14T04:15:41.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.946742+0000", 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010087299999999999 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.950609+0000", 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038667150000000002 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:119502 11.f 11:f51cce66:::benchmark_data_smithi001_228075_object1187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:41.960532+0000", 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.011928397, 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012002967, 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119502 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:41.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960532+0000", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960532+0000", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960530+0000", 2024-09-14T04:15:41.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960537+0000", 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5050000000000004e-06 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960551+0000", 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.417e-05 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960558+0000", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0269999999999998e-06 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960611+0000", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3060999999999999e-05 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960757+0000", 2024-09-14T04:15:41.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014594799999999999 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.960884+0000", 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000126599 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.964781+0000", 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038973929999999999 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.986 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:119504 11.f 11:f3cdeae4:::benchmark_data_smithi001_228075_object1189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:41.965610+0000", 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.006850086, 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069688270000000004, 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119504 2024-09-14T04:15:41.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965610+0000", 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965610+0000", 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:41.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965609+0000", 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965615+0000", 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0519999999999998e-06 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965627+0000", 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2029e-05 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965632+0000", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5740000000000003e-06 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965694+0000", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2104e-05 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965830+0000", 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135866 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.965962+0000", 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013156800000000001 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.969088+0000", 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031260630000000001 2024-09-14T04:15:41.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:119505 11.3 11:c25f0838:::benchmark_data_smithi001_228075_object1190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:41.967429+0000", 2024-09-14T04:15:41.992 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0050306459999999997, 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051921220000000004, 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119505 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:15:41.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967429+0000", 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967429+0000", 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967429+0000", 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967433+0000", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0289999999999999e-06 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967442+0000", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5910000000000002e-06 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967448+0000", 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8529999999999997e-06 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967506+0000", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7516999999999998e-05 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967684+0000", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017836800000000001 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.967845+0000", 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000160581 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:15:41.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:41.971921+0000", 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040762460000000004 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 4 2024-09-14T04:15:41.997 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:15:42.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:42.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:42.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:42.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:42 smithi001 podman[238078]: 2024-09-14 04:15:41.962568187 +0000 UTC m=+0.570179984 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:15:42.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:42 smithi001 podman[238078]: 2024-09-14 04:15:42.130387575 +0000 UTC m=+0.737999366 container create 6cdadb1d8e0867a0092dddbea0603d12834e662de0c15df789814e06047e48fc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-14T04:15:42.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:42.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:42.356 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:15:42.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:42.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:42.623 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:42.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:42 smithi001 ceph-mon[107399]: pgmap v925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 45 MiB/s wr, 731 op/s; 47770/349584 objects degraded (13.665%); 2522/349584 objects misplaced (0.721%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:15:42.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:42.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:42 smithi001 ceph-mon[112165]: pgmap v925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 45 MiB/s wr, 731 op/s; 47770/349584 objects degraded (13.665%); 2522/349584 objects misplaced (0.721%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:15:42.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:42.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:42.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:42.772 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:42 smithi012 ceph-mon[101180]: pgmap v925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 1 active+recovering+undersized+remapped, 10 active+undersized+degraded, 66 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 45 MiB/s wr, 731 op/s; 47770/349584 objects degraded (13.665%); 2522/349584 objects misplaced (0.721%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:15:42.772 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:15:42.779 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:15:42.780 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:15:42.802 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:15:42.803 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:15:42.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:15:42.830 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:15:42.830 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:15:42.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:42.864 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:15:42.864 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:15:42.901 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:42 smithi001 podman[238078]: 2024-09-14 04:15:42.888098549 +0000 UTC m=+1.495710348 container init 6cdadb1d8e0867a0092dddbea0603d12834e662de0c15df789814e06047e48fc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-14T04:15:42.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:42.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:43.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:43.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:43.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:43.148 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:43.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:43.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:42 smithi001 podman[238078]: 2024-09-14 04:15:42.900753939 +0000 UTC m=+1.508365724 container start 6cdadb1d8e0867a0092dddbea0603d12834e662de0c15df789814e06047e48fc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-14T04:15:43.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:42 smithi001 podman[238078]: 2024-09-14 04:15:42.996413959 +0000 UTC m=+1.604025749 container attach 6cdadb1d8e0867a0092dddbea0603d12834e662de0c15df789814e06047e48fc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:15:43.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:43.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:43.421 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:43.458 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:43.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:43.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:43 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:15:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:43 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:43 smithi012 ceph-mon[101180]: osdmap e262: 8 total, 7 up, 8 in 2024-09-14T04:15:43.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:43 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4147502785' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:43.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[112165]: osdmap e262: 8 total, 7 up, 8 in 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4147502785' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2482505167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:15:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[107399]: osdmap e262: 8 total, 7 up, 8 in 2024-09-14T04:15:43.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:43 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4147502785' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:43.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:43.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:43.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:43.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:43.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.008 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:44.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:15:44.036 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:15:44.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:44.150 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:44.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:44.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:44.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:44.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:44.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:44.449 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:44 smithi012 ceph-mon[101180]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 104 B/s rd, 50 MiB/s wr, 857 op/s; 47898/352848 objects degraded (13.575%); 2254/352848 objects misplaced (0.639%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:15:44.449 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 47898/352848 objects degraded (13.575%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:44.507 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:44 smithi001 ceph-mon[112165]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 104 B/s rd, 50 MiB/s wr, 857 op/s; 47898/352848 objects degraded (13.575%); 2254/352848 objects misplaced (0.639%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:15:44.507 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 47898/352848 objects degraded (13.575%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:44.508 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:44 smithi001 ceph-mon[107399]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 104 B/s rd, 50 MiB/s wr, 857 op/s; 47898/352848 objects degraded (13.575%); 2254/352848 objects misplaced (0.639%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:15:44.508 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 47898/352848 objects degraded (13.575%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:15:44.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:44.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:44.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:44.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:44.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:44.848 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:44.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:44.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:15:44 smithi001 podman[238078]: 2024-09-14 04:15:44.554781217 +0000 UTC m=+3.162393000 container died 6cdadb1d8e0867a0092dddbea0603d12834e662de0c15df789814e06047e48fc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T04:15:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:44.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:44.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:45.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:45.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:45.183 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:45.183 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:45.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:45.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:45.376 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:45.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:45.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:45.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:45.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:45.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:45.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:45.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:45.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:45.970 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:45.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:46.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:15:46.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:46.115 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:46.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service' 2024-09-14T04:15:46.216 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:46.227 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:15:46.227 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-14T04:15:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:46.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:46.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:46.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:46.399 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:46.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:46.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:46.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:15:46.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:46.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:46.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:46 smithi001 ceph-mon[107399]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 849 op/s; 47788/352518 objects degraded (13.556%); 2254/352518 objects misplaced (0.639%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:15:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:46 smithi001 ceph-mon[112165]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 849 op/s; 47788/352518 objects degraded (13.556%); 2254/352518 objects misplaced (0.639%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:15:46.735 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:46 smithi012 ceph-mon[101180]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 849 op/s; 47788/352518 objects degraded (13.556%); 2254/352518 objects misplaced (0.639%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:15:46.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:46.802 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:46.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:46.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:46.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:46.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:46] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:15:47.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:47.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:47.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:15:47.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:47.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:47.374 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:47.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:47.516 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:15:47.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:47.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:47.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:47.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:47.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:47.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:47.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:48.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:48.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:48.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:48.133 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:48.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:48.195 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:48.297 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:48.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:48.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:48.467 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:48 smithi001 ceph-mon[107399]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 944 op/s; 46247/338982 objects degraded (13.643%); 2023/338982 objects misplaced (0.597%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:48.467 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:48 smithi001 ceph-mon[112165]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 944 op/s; 46247/338982 objects degraded (13.643%); 2023/338982 objects misplaced (0.597%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:48.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:48.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:48 smithi012 ceph-mon[101180]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 944 op/s; 46247/338982 objects degraded (13.643%); 2023/338982 objects misplaced (0.597%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:48.624 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:48.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:48.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:48.864 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:48.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:48.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:48.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:49.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:49.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:49.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:49.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:49.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:49.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:49.428 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:49.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:49.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 46247/338982 objects degraded (13.643%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 46247/338982 objects degraded (13.643%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:49.568 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:49.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:49.626 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 46247/338982 objects degraded (13.643%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:49.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:49.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:49.798 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:15:49.798 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:15:49.819 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:15:49.819 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:15:49.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:49.865 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:15:49.865 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:15:49.865 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:15:49.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:49.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:49.895 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:15:49.896 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:15:49.927 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:15:49.927 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:15:49.954 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:15:49.954 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:15:49.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:49.996 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:15:49.996 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:15:50.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:50.031 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:15:50.031 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:15:50.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:50.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:50.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:50.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:50.389 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:50.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:50.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:50.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:50 smithi012 ceph-mon[101180]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 948 op/s; 45923/338010 objects degraded (13.586%); 2023/338010 objects misplaced (0.599%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3536303715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:50.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:15:50.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:50.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[107399]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 948 op/s; 45923/338010 objects degraded (13.586%); 2023/338010 objects misplaced (0.599%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:50.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3536303715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:50.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:50.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:50.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[112165]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 948 op/s; 45923/338010 objects degraded (13.586%); 2023/338010 objects misplaced (0.599%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:50.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3536303715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:15:50.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:15:50.696 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:50.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:50.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:50.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:50.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:50.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:51.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:51.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:51.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:51.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:51.228 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2024-09-14T04:15:51.228 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:15:51.228 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:15:51.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-09-14T04:15:51.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:51.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:51.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:51.382 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:51.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:51.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:51.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:51.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:51.723 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:15:51.724 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:15:51.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:51.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:51.868 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:51.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:51.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:51.975 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:51.983 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:52.002 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:52.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:52.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:52.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:52.269 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:52.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:52.399 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:15:52.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:52.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:52.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:52.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:52.649 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:52.682 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[112165]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 11 MiB/s wr, 847 op/s; 45297/334254 objects degraded (13.552%); 2023/334254 objects misplaced (0.605%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2976909045' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[107399]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 11 MiB/s wr, 847 op/s; 45297/334254 objects degraded (13.552%); 2023/334254 objects misplaced (0.605%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2976909045' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:52.823 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:52.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:52 smithi012 ceph-mon[101180]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 11 MiB/s wr, 847 op/s; 45297/334254 objects degraded (13.552%); 2023/334254 objects misplaced (0.605%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:15:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2976909045' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:15:52.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:52.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:52.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:53.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:53.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:53.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:53.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:53.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:53.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:53.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:53.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:53.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:53.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:53.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:53.659 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[107399]: osdmap e263: 8 total, 7 up, 8 in 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:53 smithi001 ceph-mon[112165]: osdmap e263: 8 total, 7 up, 8 in 2024-09-14T04:15:53.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:15:53.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:53.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:53 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:15:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:15:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:53 smithi012 ceph-mon[101180]: osdmap e263: 8 total, 7 up, 8 in 2024-09-14T04:15:53.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:53.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:53.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:54.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:54.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:54.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:54.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:54.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:54.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:54.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:54.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:54.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:54.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:54.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:54 smithi001 ceph-mon[107399]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 966 KiB/s wr, 990 op/s; 44368/324039 objects degraded (13.692%); 1766/324039 objects misplaced (0.545%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 44368/324039 objects degraded (13.692%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:54 smithi001 ceph-mon[112165]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 966 KiB/s wr, 990 op/s; 44368/324039 objects degraded (13.692%); 1766/324039 objects misplaced (0.545%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 44368/324039 objects degraded (13.692%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:54.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:54.775 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:54 smithi012 ceph-mon[101180]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 966 KiB/s wr, 990 op/s; 44368/324039 objects degraded (13.692%); 1766/324039 objects misplaced (0.545%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 44368/324039 objects degraded (13.692%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:54.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:54.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:54.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:55.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:55.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:55.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:55.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:55.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:55.330 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:55.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:55.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:55.474 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:55.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:55.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:55.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:55.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:55.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:55.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:55.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:55.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:55.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:56.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:56.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:56.161 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:56.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:56.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:56.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:56.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:56.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:56.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:56.687 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:56.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:56 smithi001 ceph-mon[107399]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 435 KiB/s wr, 995 op/s; 44036/323043 objects degraded (13.632%); 1766/323043 objects misplaced (0.547%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:56 smithi001 ceph-mon[112165]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 435 KiB/s wr, 995 op/s; 44036/323043 objects degraded (13.632%); 1766/323043 objects misplaced (0.547%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:56.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.730 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:56.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:56.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:56 smithi012 ceph-mon[101180]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 435 KiB/s wr, 995 op/s; 44036/323043 objects degraded (13.632%); 1766/323043 objects misplaced (0.547%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:15:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:56.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:56.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:15:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:15:56] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.51.0" 2024-09-14T04:15:56.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:57.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:57.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:57.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:57.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:57.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:57.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:57.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:57.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:15:57.667 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:57.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:57.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:57.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:57.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:57.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:58.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:58.079 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:135729 11.1 e263/244 11:8cb5368f:::benchmark_data_smithi001_228075_object17414:head v 263'8405, mlcod=263'8401)", 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:58.068528+0000", 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0095031950000000007, 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0095300209999999996, 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:15:58.080 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068528+0000", 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068528+0000", 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068529+0000", 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6499999999999998e-07 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.081 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068535+0000", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.84e-06 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068552+0000", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6562000000000002e-05 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.068558+0000", 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7349999999999999e-06 2024-09-14T04:15:58.082 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.071073+0000", 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025143140000000001 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.071120+0000", 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7009999999999999e-05 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.083 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:135739 11.9 e263/252 11:9428d623:::benchmark_data_smithi001_228075_object17424:head v 263'8379, mlcod=249'2563)", 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:58.072861+0000", 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0051710940000000002, 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005253944, 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072861+0000", 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:15:58.084 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072861+0000", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072862+0000", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0890000000000001e-06 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072869+0000", 2024-09-14T04:15:58.085 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0809999999999996e-06 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072886+0000", 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7428e-05 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.072894+0000", 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6109999999999996e-06 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.086 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.075706+0000", 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028126969999999999 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:58.075733+0000", 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6288999999999999e-05 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:15:58.087 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:15:58.088 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 2 2024-09-14T04:15:58.088 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:15:58.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:15:58.185 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:15:58.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:58.323 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:58.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:58.454 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:58.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:58.501 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:15:58.556 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:15:58.600 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:15:58.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:58 smithi001 ceph-mon[107399]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42440/308556 objects degraded (13.754%); 1497/308556 objects misplaced (0.485%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:15:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:58 smithi001 ceph-mon[112165]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42440/308556 objects degraded (13.754%); 1497/308556 objects misplaced (0.485%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:15:58.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:58.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:58.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:15:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:58 smithi012 ceph-mon[101180]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42440/308556 objects degraded (13.754%); 1497/308556 objects misplaced (0.485%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:15:58.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:58.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:15:58.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:moved 0 / 297 (0%) 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:avg 42.4286 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 8.22267 -> 8.22267 (expected baseline 6.03053) 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.6 with 23 -> 23 pgs (0.542088 -> 0.542088 * mean) 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 45 -> 45 pgs (1.06061 -> 1.06061 * mean) 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:15:58.998 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:15:58.999 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 0.05 2024-09-14T04:15:58.999 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:15:58.999 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0344 2024-09-14T04:15:58.999 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.3440 2024-09-14T04:15:58.999 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:15:59.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:59.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:59.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:59.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:59.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:15:59.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:59.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:15:59.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:15:59.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:15:59.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:59.566 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:59.657 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:15:59.667 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:15:59.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:15:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 42440/308556 objects degraded (13.754%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:59.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:15:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 42440/308556 objects degraded (13.754%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:59.736 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:15:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:15:59.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:15:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:15:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 42440/308556 objects degraded (13.754%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:15:59.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:15:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:15:59.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:15:59.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:00.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:00.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:00.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:00.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:00.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:00.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:00.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:00.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:00.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:00.575 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:16:00.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-09-14T04:16:00.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:00.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:00.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:00.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:00 smithi001 ceph-mon[112165]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:00 smithi001 ceph-mon[112165]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42134/307638 objects degraded (13.696%); 1497/307638 objects misplaced (0.487%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:00 smithi001 ceph-mon[107399]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:00 smithi001 ceph-mon[107399]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42134/307638 objects degraded (13.696%); 1497/307638 objects misplaced (0.487%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:00.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:00 smithi012 ceph-mon[101180]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:00 smithi012 ceph-mon[101180]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 42134/307638 objects degraded (13.696%); 1497/307638 objects misplaced (0.487%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:00.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:00.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:00.985 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:00.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:01.004 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:01.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:01.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:01.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:01.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:01.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:01.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:01.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:01.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:01.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:01.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:01.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:01.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:01.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:01.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:01.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:01.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:01.949 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:01.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:02.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:02.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:02.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:02.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:02.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:02.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:02.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.522 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:02.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:02.645 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:02 smithi001 ceph-mon[107399]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41455/303510 objects degraded (13.659%); 1497/303510 objects misplaced (0.493%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:02 smithi001 ceph-mon[112165]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41455/303510 objects degraded (13.659%); 1497/303510 objects misplaced (0.493%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:02.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:02.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:02 smithi012 ceph-mon[101180]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 41455/303510 objects degraded (13.659%); 1497/303510 objects misplaced (0.493%); 3.3 MiB/s, 52 objects/s recovering 2024-09-14T04:16:02.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:02.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:02.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:03.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:03.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:03.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:03.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:03.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:03.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:03.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:03.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:03.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:03.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:03.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:03.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:03.725 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:03.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:03.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:03.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:03.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:03.999 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:03.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:04.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:04.125 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:04.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:04.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:04.333 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:04.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:04.417 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:04.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:04.515 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:04 smithi001 ceph-mon[107399]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 40505/293103 objects degraded (13.819%); 1228/293103 objects misplaced (0.419%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:04.515 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 40505/293103 objects degraded (13.819%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:04.516 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:04 smithi001 ceph-mon[112165]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 40505/293103 objects degraded (13.819%); 1228/293103 objects misplaced (0.419%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:04.516 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 40505/293103 objects degraded (13.819%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:04.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:04.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:04.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:04.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:04.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:04.839 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:04.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:04.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:04 smithi012 ceph-mon[101180]: pgmap v938: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 40505/293103 objects degraded (13.819%); 1228/293103 objects misplaced (0.419%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 40505/293103 objects degraded (13.819%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:04.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:05.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:05.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:05.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:05.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:05.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:05.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:05.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:05.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:05.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:05.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:05.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:05.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:05.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:05.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:05.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:05.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:05.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:06.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:06.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:06.140 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:06.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[107399]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 40170/292098 objects degraded (13.752%); 1228/292098 objects misplaced (0.420%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:05 smithi001 ceph-mon[112165]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 40170/292098 objects degraded (13.752%); 1228/292098 objects misplaced (0.420%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:16:06.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:06.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:05 smithi012 ceph-mon[101180]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 40170/292098 objects degraded (13.752%); 1228/292098 objects misplaced (0.420%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:16:06.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:06.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:06.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:06.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:06.696 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:06.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:06.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:06.857 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:06] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.51.0" 2024-09-14T04:16:06.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:06.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:06.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:07.021 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:07.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:07.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:07.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:07.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:07.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:07.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:07.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:07.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:07.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:07.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:07.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:07.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:07.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:08.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:08.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.141 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:08.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:08.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:08.287 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:08.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:08.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:08.438 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:08 smithi001 ceph-mon[107399]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 38662/278037 objects degraded (13.905%); 830/278037 objects misplaced (0.299%); 4.9 MiB/s, 77 objects/s recovering 2024-09-14T04:16:08.438 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:08 smithi001 ceph-mon[112165]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 38662/278037 objects degraded (13.905%); 830/278037 objects misplaced (0.299%); 4.9 MiB/s, 77 objects/s recovering 2024-09-14T04:16:08.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:08.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:08 smithi012 ceph-mon[101180]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 38662/278037 objects degraded (13.905%); 830/278037 objects misplaced (0.299%); 4.9 MiB/s, 77 objects/s recovering 2024-09-14T04:16:08.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:08.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:08.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:08.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:08.889 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:16:08.890 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:16:08.926 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:16:08.926 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:16:08.927 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:16:08.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:08.961 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:16:08.961 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:16:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:08.998 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:08.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:08.999 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:16:08.999 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:16:09.031 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:16:09.031 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:16:09.067 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:16:09.068 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:16:09.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:09.100 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:16:09.100 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:16:09.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:09.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:09.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:09.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:09.405 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:09.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:09.540 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1011790199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 38662/278037 objects degraded (13.905%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:09.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:09.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:09.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1011790199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:09.694 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 38662/278037 objects degraded (13.905%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:09.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1011790199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:09.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 38662/278037 objects degraded (13.905%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:09.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:09.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:09.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:09.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:09.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:10.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:10.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:16:10.171 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:16:10.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:10.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:10.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:10.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:10.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:10.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:10 smithi001 ceph-mon[107399]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 38382/277197 objects degraded (13.846%); 830/277197 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:10.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:10 smithi001 ceph-mon[112165]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 38382/277197 objects degraded (13.846%); 830/277197 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:10.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.516 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:10.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:10 smithi012 ceph-mon[101180]: pgmap v941: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 38382/277197 objects degraded (13.846%); 830/277197 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:10.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:10.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:10.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:10.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:10.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:10.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:10.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:11.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:11.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:11.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:11.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:11.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:11.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:11.332 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:11.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:11.471 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:11.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:11.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:11.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:11.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:11.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:11.893 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:11.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:11.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:12.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:12.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:16:12.179 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:12.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:12.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:12.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:12.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:12.478 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:12 smithi001 ceph-mon[107399]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 37747/273351 objects degraded (13.809%); 830/273351 objects misplaced (0.304%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:12.478 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:12 smithi001 ceph-mon[112165]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 37747/273351 objects degraded (13.809%); 830/273351 objects misplaced (0.304%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:12.478 INFO:teuthology.orchestra.run.smithi001.stdout:moved 0 / 297 (0%) 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:avg 42.4286 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 8.22267 -> 8.22267 (expected baseline 6.03053) 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.6 with 23 -> 23 pgs (0.542088 -> 0.542088 * mean) 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 45 -> 45 pgs (1.06061 -> 1.06061 * mean) 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 1 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0319 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.3193 2024-09-14T04:16:12.479 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:16:12.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:12.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:12 smithi012 ceph-mon[101180]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 37747/273351 objects degraded (13.809%); 830/273351 objects misplaced (0.304%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:16:12.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:12.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:12.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:12.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:12.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:12.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:12.988 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:12.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:13.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:13.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:13.214 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:13.237 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:16:13.237 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:16:13.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:13.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:13.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:13 smithi001 ceph-mon[107399]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:13 smithi001 ceph-mon[112165]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:13.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:13.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:13.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:13.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:13.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:13.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:13.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:13 smithi012 ceph-mon[101180]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:13.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:13.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:13.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:14.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:14.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:14.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:14.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:14.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:14.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:14.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:14.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:14.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:14.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:14.656 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:14 smithi001 ceph-mon[107399]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 36831/263202 objects degraded (13.993%); 394/263202 objects misplaced (0.150%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:16:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 36831/263202 objects degraded (13.993%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:14 smithi001 ceph-mon[112165]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 36831/263202 objects degraded (13.993%); 394/263202 objects misplaced (0.150%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:16:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 36831/263202 objects degraded (13.993%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:14.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:14.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:14 smithi012 ceph-mon[101180]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 36831/263202 objects degraded (13.993%); 394/263202 objects misplaced (0.150%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:16:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 36831/263202 objects degraded (13.993%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:14.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:14.906 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:16:14.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:14.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:15.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:15.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:15.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:15.212 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:15.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:15.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.356 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:16:15.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-09-14T04:16:15.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:15.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:15.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:15.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:15.638 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:15.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:15.776 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:15.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:15.908 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:15.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:16.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:16.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:16.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:16.169 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:16.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:16.212 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:16.241 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:16.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:16.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:16.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:16.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:16 smithi001 ceph-mon[107399]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 36546/262347 objects degraded (13.930%); 394/262347 objects misplaced (0.150%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:16:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:16 smithi001 ceph-mon[112165]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 36546/262347 objects degraded (13.930%); 394/262347 objects misplaced (0.150%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:16:16.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:16.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:16.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:16.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:16.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:16.839 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:16] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.51.0" 2024-09-14T04:16:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:16 smithi012 ceph-mon[101180]: pgmap v944: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 36546/262347 objects degraded (13.930%); 394/262347 objects misplaced (0.150%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:16:16.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:16.887 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:16.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:16.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.028 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:17.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:17.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:17.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:17.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:17.314 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:17.372 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:16:17.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:17.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:17.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:17.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:17.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:17.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:17.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:17 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:17 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:17.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:17.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:17 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:17.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:17.968 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:17.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:18.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:18.094 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:18.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:18.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:18.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:18.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:18.342 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:18.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:18.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:18.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:18.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:18.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:18.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[112165]: pgmap v945: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 35073/248874 objects degraded (14.093%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[112165]: osdmap e264: 8 total, 7 up, 8 in 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[107399]: pgmap v945: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 35073/248874 objects degraded (14.093%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:18.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:18 smithi001 ceph-mon[107399]: osdmap e264: 8 total, 7 up, 8 in 2024-09-14T04:16:18.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:18.765 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:18 smithi012 ceph-mon[101180]: pgmap v945: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 35073/248874 objects degraded (14.093%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:16:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:18 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1282334780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:18 smithi012 ceph-mon[101180]: osdmap e264: 8 total, 7 up, 8 in 2024-09-14T04:16:18.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:18.899 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:18.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:18.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:19.001 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:19.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:19.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:19.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:19.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:19.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:19.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:19.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:19.583 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:19.651 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:19 smithi001 ceph-mon[112165]: osdmap e265: 8 total, 7 up, 8 in 2024-09-14T04:16:19.651 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 35073/248874 objects degraded (14.093%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:19.652 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:19 smithi001 ceph-mon[107399]: osdmap e265: 8 total, 7 up, 8 in 2024-09-14T04:16:19.652 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 35073/248874 objects degraded (14.093%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:19.685 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:19.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:19.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:19 smithi012 ceph-mon[101180]: osdmap e265: 8 total, 7 up, 8 in 2024-09-14T04:16:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 35073/248874 objects degraded (14.093%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:19.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:19.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:19.956 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:16:19.956 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:16:19.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:19.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:20.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:20.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:20.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:20.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:20.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:20.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:20.490 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:16:20.490 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:16:20.520 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:16:20.520 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:16:20.520 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:20 smithi001 ceph-mon[112165]: pgmap v948: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 34801/248058 objects degraded (14.029%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:16:20.520 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:20.521 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:20 smithi001 ceph-mon[107399]: pgmap v948: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 34801/248058 objects degraded (14.029%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:16:20.521 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:20.521 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:16:20.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:20.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:20.557 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:16:20.557 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:16:20.585 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:16:20.586 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:16:20.622 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:16:20.622 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:16:20.648 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:16:20.648 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:16:20.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:20.675 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:16:20.675 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:16:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:20.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:20.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:20.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:20.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:20 smithi012 ceph-mon[101180]: pgmap v948: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 34801/248058 objects degraded (14.029%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:16:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:20.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:20.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:20.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:20.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:21.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:21.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.116 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:21.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:21.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:16:21.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:21.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:21.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1193037787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1193037787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:21.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:21.522 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:21.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:21.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:21.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:21.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1193037787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:21.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:21.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:21.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:22.042 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:22.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:22.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:22.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:22.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:22.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:22.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:22.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:22.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:22.632 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:22.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:22 smithi001 ceph-mon[107399]: pgmap v949: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 34290/244977 objects degraded (13.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:22 smithi001 ceph-mon[112165]: pgmap v949: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 34290/244977 objects degraded (13.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:22.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:22.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:22 smithi012 ceph-mon[101180]: pgmap v949: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 34290/244977 objects degraded (13.997%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:16:22.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:22.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:22.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:23.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:23.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:23.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:23.216 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:23.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:23.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:16:23.454 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:16:23.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:23.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:23.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:23.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:23.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:23.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:23.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:23.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:23.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:23.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:23.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:24.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:24.020 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:24.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:24.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:24.210 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:24.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:24.343 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:24.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:16:24.429 INFO:teuthology.orchestra.run.smithi001.stdout:moved 3 / 297 (1.0101%) 2024-09-14T04:16:24.429 INFO:teuthology.orchestra.run.smithi001.stdout:avg 42.4286 2024-09-14T04:16:24.429 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 8.22267 -> 7.94111 (expected baseline 6.03053) 2024-09-14T04:16:24.429 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.6 with 23 -> 23 pgs (0.542088 -> 0.542088 * mean) 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 45 -> 45 pgs (1.06061 -> 1.06061 * mean) 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 0.05 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0291 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0320 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 1.0000 -> 0.9500 2024-09-14T04:16:24.430 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 weight 1.0000 -> 0.9500 2024-09-14T04:16:24.431 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:16:24.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:24.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:24.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:24.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:24.624 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:24 smithi001 ceph-mon[107399]: pgmap v950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31809/236052 objects degraded (13.475%); 1500/236052 objects misplaced (0.635%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:16:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 31809/236052 objects degraded (13.475%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:24 smithi001 ceph-mon[112165]: pgmap v950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31809/236052 objects degraded (13.475%); 1500/236052 objects misplaced (0.635%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:16:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 31809/236052 objects degraded (13.475%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:24.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:24 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:16:24.423+0000 7fc9b3f23640 -1 mgr.server reply reply (2) No such file or directory no change 2024-09-14T04:16:24.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.769 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:24.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:24 smithi012 ceph-mon[101180]: pgmap v950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31809/236052 objects degraded (13.475%); 1500/236052 objects misplaced (0.635%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:16:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 31809/236052 objects degraded (13.475%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:24.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:24.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:24.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:25.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:25.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:25.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:25.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:25.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:25.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:25.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:25.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:25.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:25.455 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:16:25.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:25.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:25.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:25 smithi001 ceph-mon[107399]: from='client.45482 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:25.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:25 smithi001 ceph-mon[112165]: from='client.45482 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:25.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:25.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:25.683 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:16:25.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 3.0 2024-09-14T04:16:25.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:25.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:25.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:25 smithi012 ceph-mon[101180]: from='client.45482 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:25.888 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:25.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:25.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:26.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:26.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:26.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:26.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:26.264 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:26.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:26.410 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:26.454 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:26.556 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:26.588 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:26.632 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:26.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:26 smithi001 ceph-mon[107399]: pgmap v951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 31502/235131 objects degraded (13.398%); 1500/235131 objects misplaced (0.638%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:16:26.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:26 smithi001 ceph-mon[112165]: pgmap v951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 31502/235131 objects degraded (13.398%); 1500/235131 objects misplaced (0.638%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:16:26.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:26.740 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:26.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:26 smithi012 ceph-mon[101180]: pgmap v951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 31502/235131 objects degraded (13.398%); 1500/235131 objects misplaced (0.638%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:16:26.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:26.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:26] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.51.0" 2024-09-14T04:16:26.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:27.015 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:27.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:27.156 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:27.161 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:27.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:27.207 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:16:27.207 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163666 11.f e265/260 11:fa74ef90:::benchmark_data_smithi001_228075_object45351:head v 265'10352, mlcod=265'10350)", 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.179919+0000", 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.023370927999999999, 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.023400293999999999, 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179919+0000", 2024-09-14T04:16:27.208 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179919+0000", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179920+0000", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.269e-06 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179929+0000", 2024-09-14T04:16:27.209 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999992e-06 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179946+0000", 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7218999999999999e-05 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.179954+0000", 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5860000000000005e-06 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.210 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181576+0000", 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0016217600000000001 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181598+0000", 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1716e-05 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163665 11.9 e265/264 11:934cfdb9:::benchmark_data_smithi001_228075_object45350:head v 265'10110, mlcod=265'10108)", 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.181559+0000", 2024-09-14T04:16:27.211 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.021730987, 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021816855999999999, 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181559+0000", 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.212 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181559+0000", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181559+0000", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8500000000000002e-07 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181564+0000", 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8709999999999996e-06 2024-09-14T04:16:27.213 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181571+0000", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5680000000000002e-06 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.181578+0000", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3210000000000001e-06 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.183238+0000", 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001660316 2024-09-14T04:16:27.214 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.183263+0000", 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5397000000000001e-05 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163668 11.9 e265/264 11:94a167b7:::benchmark_data_smithi001_228075_object45353:head v 265'10111, mlcod=265'10108)", 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.186891+0000", 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.016398705, 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016534387000000001, 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.215 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186891+0000", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186891+0000", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.216 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186895+0000", 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4100000000000001e-06 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186903+0000", 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1829999999999994e-06 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.217 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186922+0000", 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8281999999999999e-05 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.186930+0000", 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2300000000000008e-06 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.191434+0000", 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045043080000000003 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.218 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.191454+0000", 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9913999999999999e-05 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163674 11.7 e265/244 11:e1a0326b:::benchmark_data_smithi001_228075_object45359:head v 265'10258, mlcod=265'10254)", 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.187971+0000", 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.015318635000000001, 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015501668, 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.219 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187971+0000", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187971+0000", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.220 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187971+0000", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1500000000000005e-07 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187976+0000", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5950000000000001e-06 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187986+0000", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6330000000000008e-06 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.187992+0000", 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6880000000000002e-06 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.188050+0000", 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7593999999999998e-05 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.188068+0000", 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8454999999999999e-05 2024-09-14T04:16:27.222 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163676 11.9 e265/264 11:9d29f0f2:::benchmark_data_smithi001_228075_object45361:head v 265'10112, mlcod=265'10108)", 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.193829+0000", 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0094602869999999995, 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0096914859999999992, 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193829+0000", 2024-09-14T04:16:27.223 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193829+0000", 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193831+0000", 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.285e-06 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.224 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193837+0000", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1340000000000001e-06 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193849+0000", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2184000000000001e-05 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.193856+0000", 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0709999999999997e-06 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.225 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.195121+0000", 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012648360000000001 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.195141+0000", 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0210000000000001e-05 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163677 11.f e265/260 11:f15ae77c:::benchmark_data_smithi001_228075_object45362:head v 265'10353, mlcod=265'10350)", 2024-09-14T04:16:27.226 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.194608+0000", 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0086812980000000005, 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.008959669, 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194608+0000", 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.227 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194608+0000", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194609+0000", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1100000000000004e-07 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194617+0000", 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4410000000000004e-06 2024-09-14T04:16:27.228 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194628+0000", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1912e-05 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.194635+0000", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7830000000000001e-06 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.197617+0000", 2024-09-14T04:16:27.229 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029812469999999998 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.197638+0000", 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1047999999999999e-05 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163679 11.f e265/260)", 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.199104+0000", 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0041853300000000001, 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045078820000000004, 2024-09-14T04:16:27.230 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199104+0000", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199104+0000", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.231 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199105+0000", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0279999999999999e-06 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199113+0000", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.763e-06 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199128+0000", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4883e-05 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.199135+0000", 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.216e-06 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163678 11.9 e265/264 11:96c994e8:::benchmark_data_smithi001_228075_object45363:head v 265'10113, mlcod=265'10108)", 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.200153+0000", 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0031367830000000002, 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034994890000000002, 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200153+0000", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200153+0000", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200154+0000", 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.23e-07 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200159+0000", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3260000000000003e-06 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200171+0000", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1829e-05 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.200178+0000", 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3030000000000002e-06 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.202510+0000", 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023318800000000002 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.202531+0000", 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1304000000000001e-05 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:163682 11.6 e265/260 11:62521900:::benchmark_data_smithi001_228075_object45367:head v 265'10257, mlcod=265'10255)", 2024-09-14T04:16:27.236 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:27.203046+0000", 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.00024321799999999999, 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00065433799999999997, 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203046+0000", 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:27.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203046+0000", 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203047+0000", 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.153e-06 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203054+0000", 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4289999999999998e-06 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203067+0000", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2908000000000001e-05 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203074+0000", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3370000000000002e-06 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203128+0000", 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4014000000000002e-05 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:27.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:27.203150+0000", 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1401999999999999e-05 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 9 2024-09-14T04:16:27.240 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:16:27.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:27.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:27.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:27.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:27.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:27.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:27.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:27.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:27.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:27.843 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:27.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:27.978 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:27.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:28.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:28.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:28.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:28.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:28.347 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:28.392 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:28.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:28.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:28.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:16:28.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:28.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:28 smithi001 ceph-mon[107399]: pgmap v952: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 30124/221511 objects degraded (13.599%); 1214/221511 objects misplaced (0.548%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:16:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:28 smithi001 ceph-mon[112165]: pgmap v952: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 30124/221511 objects degraded (13.599%); 1214/221511 objects misplaced (0.548%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:16:28.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:28.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:28 smithi012 ceph-mon[101180]: pgmap v952: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 30124/221511 objects degraded (13.599%); 1214/221511 objects misplaced (0.548%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:16:28.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:28.958 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:28.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:29.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:29.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:29.192 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:29.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:29.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:29.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:29.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:29.522 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:29.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:29.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 30124/221511 objects degraded (13.599%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 30124/221511 objects degraded (13.599%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:29.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:29.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 30124/221511 objects degraded (13.599%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:29.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:29.928 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:29.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:30.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:30.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:30.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:30.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:30.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:30.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.350 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:30.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:30.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:30.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:30.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:30 smithi001 ceph-mon[107399]: pgmap v953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 29818/220593 objects degraded (13.517%); 1214/220593 objects misplaced (0.550%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:16:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:30 smithi001 ceph-mon[112165]: pgmap v953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 29818/220593 objects degraded (13.517%); 1214/220593 objects misplaced (0.550%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:16:30.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:30.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:30 smithi012 ceph-mon[101180]: pgmap v953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 29818/220593 objects degraded (13.517%); 1214/220593 objects misplaced (0.550%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:16:30.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:30.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:30.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:30.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:31.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:31.057 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:31.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:31.195 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:31.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:31.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:31.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:31.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:31.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:31.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:31.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:31.750 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:31.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:31.893 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:31.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:31.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:32.026 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:32.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:32.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:32.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:32.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:32.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:32.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:32.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:32 smithi001 ceph-mon[107399]: pgmap v954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 29236/216933 objects degraded (13.477%); 1214/216933 objects misplaced (0.560%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:16:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:32 smithi001 ceph-mon[112165]: pgmap v954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 29236/216933 objects degraded (13.477%); 1214/216933 objects misplaced (0.560%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:16:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.738 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:32.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:32.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:32 smithi012 ceph-mon[101180]: pgmap v954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 29236/216933 objects degraded (13.477%); 1214/216933 objects misplaced (0.560%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:16:32.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:32.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:32.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:33.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:33.120 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:33.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:33.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:33.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:33.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:33.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:33.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:33.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:33.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:33.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:33.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:33.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:33.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.822 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:33.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:33.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:33.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:33.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:33.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:34.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:34.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:34.246 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:34.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:34.384 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:34.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:34.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:34.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:34 smithi001 ceph-mon[107399]: pgmap v955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 28479/206826 objects degraded (13.770%); 905/206826 objects misplaced (0.438%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:16:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 28479/206826 objects degraded (13.770%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:34 smithi001 ceph-mon[112165]: pgmap v955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 28479/206826 objects degraded (13.770%); 905/206826 objects misplaced (0.438%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:16:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 28479/206826 objects degraded (13.770%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:34.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:34.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:34 smithi012 ceph-mon[101180]: pgmap v955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 28479/206826 objects degraded (13.770%); 905/206826 objects misplaced (0.438%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:16:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 28479/206826 objects degraded (13.770%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:16:34.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:34.939 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:34.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:35.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:35.082 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:35.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:35.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:35.316 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:35.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:35.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:35.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:35.574 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:35.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:35.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:35.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.735 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:35.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:35.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:35.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:35.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:35.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:36.006 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:36.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:36.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:36.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:36.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:36.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:36.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:36.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:36.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:36.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:36.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:36 smithi001 ceph-mon[107399]: pgmap v956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 28179/205926 objects degraded (13.684%); 905/205926 objects misplaced (0.439%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:16:36.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:36.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:36 smithi001 ceph-mon[112165]: pgmap v956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 28179/205926 objects degraded (13.684%); 905/205926 objects misplaced (0.439%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:16:36.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:36.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:36] "GET /metrics HTTP/1.1" 200 36617 "" "Prometheus/2.51.0" 2024-09-14T04:16:36.752 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:36 smithi012 ceph-mon[101180]: pgmap v956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 28179/205926 objects degraded (13.684%); 905/205926 objects misplaced (0.439%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:16:36.752 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:36.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:36.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:36.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:36.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:36.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:36.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:37.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:37.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:37.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:37.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:37.296 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:37.332 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:37.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:37.435 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:37.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:37.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:37.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:37.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:37.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:37.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:37.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:37.928 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:16:37.928 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:16:37.951 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:16:37.952 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:16:37.952 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:16:37.972 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:16:37.972 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:16:38.002 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:16:38.003 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:16:38.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:38.041 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:16:38.041 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:16:38.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:38.070 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:16:38.071 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:16:38.104 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:16:38.104 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:16:38.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:38.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:38.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:38.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:38.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:38.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:38.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:38.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:38.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:38.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:38 smithi001 ceph-mon[107399]: pgmap v957: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 26856/192597 objects degraded (13.944%); 597/192597 objects misplaced (0.310%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:16:38.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/96385291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:38.713 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:38 smithi001 ceph-mon[112165]: pgmap v957: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 26856/192597 objects degraded (13.944%); 597/192597 objects misplaced (0.310%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:16:38.713 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/96385291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:38.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:38.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:38 smithi012 ceph-mon[101180]: pgmap v957: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 26856/192597 objects degraded (13.944%); 597/192597 objects misplaced (0.310%); 2.4 MiB/s, 39 objects/s recovering 2024-09-14T04:16:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/96385291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:38.899 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:38.942 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:38.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:39.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:39.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:39.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:39.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:39.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:39.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:39.478 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:16:39.480 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:16:39.480 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"} : dispatch 2024-09-14T04:16:39.480 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 26856/192597 objects degraded (13.944%), 11 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:39.480 INFO:teuthology.orchestra.run.smithi001.stdout:moved 7 / 297 (2.3569%) 2024-09-14T04:16:39.480 INFO:teuthology.orchestra.run.smithi001.stdout:avg 42.4286 2024-09-14T04:16:39.480 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 8.22267 -> 7.79848 (expected baseline 6.03053) 2024-09-14T04:16:39.480 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.6 with 23 -> 24 pgs (0.542088 -> 0.565657 * mean) 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 45 -> 46 pgs (1.06061 -> 1.08418 * mean) 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0259 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0285 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 weight 1.0000 -> 0.7887 2024-09-14T04:16:39.481 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 1.0000 -> 0.7996 2024-09-14T04:16:39.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:16:39.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"} : dispatch 2024-09-14T04:16:39.482 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 26856/192597 objects degraded (13.944%), 11 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:39.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:16:39.512 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:16:39.521 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:39.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:39.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:39.654 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:16:39.655 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"} : dispatch 2024-09-14T04:16:39.655 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 26856/192597 objects degraded (13.944%), 11 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:39.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:39.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:39.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:39.899 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:39.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:40.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:40.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:40.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:40.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:40.251 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:40.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:40.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:40.397 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:40.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:40.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: pgmap v958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 26600/191829 objects degraded (13.867%); 597/191829 objects misplaced (0.311%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:16:40.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:16:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"}]': finished 2024-09-14T04:16:40.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:40 smithi012 ceph-mon[101180]: osdmap e266: 8 total, 7 up, 8 in 2024-09-14T04:16:40.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:40.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: pgmap v958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 26600/191829 objects degraded (13.867%); 597/191829 objects misplaced (0.311%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"}]': finished 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[112165]: osdmap e266: 8 total, 7 up, 8 in 2024-09-14T04:16:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: pgmap v958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 10 active+undersized+degraded, 67 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 26600/191829 objects degraded (13.867%); 597/191829 objects misplaced (0.311%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:16:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:16:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/380402630' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:16:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'52405','2':'51685'}"}]': finished 2024-09-14T04:16:40.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:40 smithi001 ceph-mon[107399]: osdmap e266: 8 total, 7 up, 8 in 2024-09-14T04:16:40.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:40.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:40.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:40.899 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:40.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:41.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:41.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:41.125 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:41.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:41.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:41.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:41.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:41.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:41.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:41.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:16:41.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:41.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:41.621 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:16:41.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-09-14T04:16:41.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:41.653 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:41 smithi001 ceph-mon[107399]: osdmap e267: 8 total, 7 up, 8 in 2024-09-14T04:16:41.654 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:41 smithi001 ceph-mon[112165]: osdmap e267: 8 total, 7 up, 8 in 2024-09-14T04:16:41.661 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:41.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:41.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.805 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:41.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:41 smithi012 ceph-mon[101180]: osdmap e267: 8 total, 7 up, 8 in 2024-09-14T04:16:41.907 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:41.942 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:41.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:42.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:42.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:42.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:42.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:42.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:42.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:42.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:42.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:16:42.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:42.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:42.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:42.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:42.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: pgmap v961: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 22398/189474 objects degraded (11.821%); 597/189474 objects misplaced (0.315%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: 11.6 starting backfill to osd.4 from (0'0,0'0] MAX to 265'10946 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: osdmap e268: 8 total, 7 up, 8 in 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 265'10926 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: pgmap v961: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 22398/189474 objects degraded (11.821%); 597/189474 objects misplaced (0.315%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:16:42.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: 11.6 starting backfill to osd.4 from (0'0,0'0] MAX to 265'10946 2024-09-14T04:16:42.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: osdmap e268: 8 total, 7 up, 8 in 2024-09-14T04:16:42.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 265'10926 2024-09-14T04:16:42.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.753 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:42.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: pgmap v961: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 22398/189474 objects degraded (11.821%); 597/189474 objects misplaced (0.315%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:16:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: 11.6 starting backfill to osd.4 from (0'0,0'0] MAX to 265'10946 2024-09-14T04:16:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: osdmap e268: 8 total, 7 up, 8 in 2024-09-14T04:16:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 265'10926 2024-09-14T04:16:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:16:42.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:42.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:42.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:43.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:43.062 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:43.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:43.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:43.240 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:43.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:43.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:43.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:43.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:43.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:43.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:43.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:43 smithi001 ceph-mon[107399]: osdmap e269: 8 total, 7 up, 8 in 2024-09-14T04:16:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:43 smithi001 ceph-mon[107399]: pgmap v964: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 21979/184215 objects degraded (11.931%); 207/184215 objects misplaced (0.112%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:16:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:43 smithi001 ceph-mon[112165]: osdmap e269: 8 total, 7 up, 8 in 2024-09-14T04:16:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:43 smithi001 ceph-mon[112165]: pgmap v964: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 21979/184215 objects degraded (11.931%); 207/184215 objects misplaced (0.112%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:16:43.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:43.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:43.827 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:16:43.827 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:16:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:43 smithi012 ceph-mon[101180]: osdmap e269: 8 total, 7 up, 8 in 2024-09-14T04:16:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:43 smithi012 ceph-mon[101180]: pgmap v964: 99 pgs: 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 9 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 21979/184215 objects degraded (11.931%); 207/184215 objects misplaced (0.112%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:16:43.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:43.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:43.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:44.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:44.047 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:44.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:44.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:44.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:44.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:44.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:44.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:44.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:44.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:44.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:44.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:44 smithi001 ceph-mon[107399]: osdmap e270: 8 total, 7 up, 8 in 2024-09-14T04:16:44.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 21979/184215 objects degraded (11.931%), 10 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:44.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:44 smithi001 ceph-mon[112165]: osdmap e270: 8 total, 7 up, 8 in 2024-09-14T04:16:44.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 21979/184215 objects degraded (11.931%), 10 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:44.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:44.739 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:44.841 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:44 smithi012 ceph-mon[101180]: osdmap e270: 8 total, 7 up, 8 in 2024-09-14T04:16:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 21979/184215 objects degraded (11.931%), 10 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:44.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:44.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:44.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:45.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:45.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:45.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:45.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:45.310 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:45.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:45.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:45.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:45.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:45.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:45.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:45.733 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:45.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:45 smithi012 ceph-mon[101180]: pgmap v966: 99 pgs: 2 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 21579/183726 objects degraded (11.745%); 207/183726 objects misplaced (0.113%); 7.3 MiB/s, 117 objects/s recovering 2024-09-14T04:16:45.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:45.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:45.963 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:45 smithi001 ceph-mon[112165]: pgmap v966: 99 pgs: 2 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 21579/183726 objects degraded (11.745%); 207/183726 objects misplaced (0.113%); 7.3 MiB/s, 117 objects/s recovering 2024-09-14T04:16:45.963 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:45 smithi001 ceph-mon[107399]: pgmap v966: 99 pgs: 2 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 21579/183726 objects degraded (11.745%); 207/183726 objects misplaced (0.113%); 7.3 MiB/s, 117 objects/s recovering 2024-09-14T04:16:45.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:46.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:46.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:16:46.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:46.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:46.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:46.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:46.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:46.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:46.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:46.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:46.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:46.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:46.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:46.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:46.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:46] "GET /metrics HTTP/1.1" 200 36617 "" "Prometheus/2.51.0" 2024-09-14T04:16:46.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:46.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:47.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:47.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:47.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:47.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:47.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:47.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:47.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:47.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:47.541 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:47.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:47.630 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:47.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:47.794 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:47.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:47.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:47.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:47.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:48.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:48.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:48.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:48.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:48.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:48.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:48.430 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:48 smithi001 ceph-mon[112165]: pgmap v967: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 24039/171312 objects degraded (14.032%); 8.0 MiB/s, 133 objects/s recovering 2024-09-14T04:16:48.430 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:48 smithi001 ceph-mon[107399]: pgmap v967: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 24039/171312 objects degraded (14.032%); 8.0 MiB/s, 133 objects/s recovering 2024-09-14T04:16:48.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:48.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:48 smithi012 ceph-mon[101180]: pgmap v967: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 24039/171312 objects degraded (14.032%); 8.0 MiB/s, 133 objects/s recovering 2024-09-14T04:16:48.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:48.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:48.691 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:16:48.691 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:16:48.720 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:16:48.721 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:16:48.721 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:16:48.750 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:16:48.751 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:16:48.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:48.784 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:16:48.784 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:16:48.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:48.814 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:16:48.815 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:16:48.841 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:16:48.841 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:16:48.875 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:16:48.875 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:16:48.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:48.945 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:48.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:49.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:49.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:49.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:49.224 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:49.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:49.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:49.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:49.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.512 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:49.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:49.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3076255455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:49.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 24039/171312 objects degraded (14.032%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:49.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3076255455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:49.581 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 24039/171312 objects degraded (14.032%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:49.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:49.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:49.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:moved 0 / 297 (0%) 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:avg 42.4286 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 7.79848 -> 7.79848 (expected baseline 6.03053) 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.6 with 24 -> 24 pgs (0.565657 -> 0.565657 * mean) 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 46 -> 46 pgs (1.08418 -> 1.08418 * mean) 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0243 2024-09-14T04:16:49.787 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.2430 2024-09-14T04:16:49.788 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:16:49.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3076255455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:16:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 24039/171312 objects degraded (14.032%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:49.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:49.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:49.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:49.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:50.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:50.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:50.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:50.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:50.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:50.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:50.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.478 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:50.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:50.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:50.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:50.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:50.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:50 smithi001 ceph-mon[107399]: pgmap v968: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23181/170439 objects degraded (13.601%); 8.5 MiB/s, 139 objects/s recovering 2024-09-14T04:16:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:50 smithi001 ceph-mon[112165]: pgmap v968: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23181/170439 objects degraded (13.601%); 8.5 MiB/s, 139 objects/s recovering 2024-09-14T04:16:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:50.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:50 smithi012 ceph-mon[101180]: pgmap v968: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23181/170439 objects degraded (13.601%); 8.5 MiB/s, 139 objects/s recovering 2024-09-14T04:16:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:50.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:50.891 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:50.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:50.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:51.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:16:51.091 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:16:51.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:51.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:51.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:51.280 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:51.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:51.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:51.420 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:51.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:51.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:51.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:51.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:51.639 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:51 smithi001 ceph-mon[107399]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:51.640 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:51 smithi001 ceph-mon[112165]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:51.687 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:51.725 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:51.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:51 smithi012 ceph-mon[101180]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:16:51.862 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:51.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:51.964 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:52.000 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:52.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:52.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:52.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:52.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:52.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:52.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:52.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:52.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:52.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:52.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:52.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:52 smithi001 ceph-mon[107399]: pgmap v969: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22613/166992 objects degraded (13.541%); 7.5 MiB/s, 123 objects/s recovering 2024-09-14T04:16:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:52 smithi001 ceph-mon[112165]: pgmap v969: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22613/166992 objects degraded (13.541%); 7.5 MiB/s, 123 objects/s recovering 2024-09-14T04:16:52.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:52.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:52 smithi012 ceph-mon[101180]: pgmap v969: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22613/166992 objects degraded (13.541%); 7.5 MiB/s, 123 objects/s recovering 2024-09-14T04:16:52.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:52.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:52.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:53.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:53.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:16:53.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:53.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:53.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:53.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:53.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:53.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:53.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:53.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:53.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:53.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:53.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:53.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:53.942 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:53.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:54.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:54.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:54.147 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:16:54.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:54.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:54.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:54.356 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:54.383 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:16:54.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:54.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:54.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:54.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[107399]: pgmap v970: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 21511/157335 objects degraded (13.672%); 6.6 MiB/s, 130 objects/s recovering 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 21511/157335 objects degraded (13.672%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[112165]: pgmap v970: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 21511/157335 objects degraded (13.672%); 6.6 MiB/s, 130 objects/s recovering 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 21511/157335 objects degraded (13.672%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:54.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:54.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:54 smithi012 ceph-mon[101180]: pgmap v970: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 21511/157335 objects degraded (13.672%); 6.6 MiB/s, 130 objects/s recovering 2024-09-14T04:16:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:16:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 21511/157335 objects degraded (13.672%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:54.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:54.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:54.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:54.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:55.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:55.074 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:55.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:55.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:55.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:55.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:55.352 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:55.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:55.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:55.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:55.593 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:55.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[107399]: osdmap e271: 8 total, 7 up, 8 in 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:55.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:55 smithi001 ceph-mon[112165]: osdmap e271: 8 total, 7 up, 8 in 2024-09-14T04:16:55.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85146 10.2 10:4e8e28f3:::benchmark_data_smithi001_190881_object85145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 426.52776540299999, 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1709060940000002, 2024-09-14T04:16:55.739 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85146 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.740 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207175+0000", 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.813e-06 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207199+0000", 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4312999999999999e-05 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207216+0000", 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6694999999999999e-05 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207223+0000", 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.08e-06 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207276+0000", 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3038999999999998e-05 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207427+0000", 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015100000000000001 2024-09-14T04:16:55.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207565+0000", 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013732699999999999 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.744 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.745 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223766+0000", 2024-09-14T04:16:55.745 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016201706 2024-09-14T04:16:55.745 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.745 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.745 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223775+0000", 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5890000000000003e-06 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214581+0000", 2024-09-14T04:16:55.746 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99080585499999996 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214598+0000", 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7051e-05 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.747 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378012+0000", 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1634141140000001 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378079+0000", 2024-09-14T04:16:55.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7512e-05 2024-09-14T04:16:55.749 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.749 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.749 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85153 10.2 10:4ace6224:::benchmark_data_smithi001_190881_object85152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 426.51335496399997, 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1713794549999998, 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.750 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85153 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221586+0000", 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9290000000000001e-06 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221614+0000", 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8419999999999999e-05 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221629+0000", 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4602e-05 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221638+0000", 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8959999999999992e-06 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223951+0000", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023134240000000001 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223958+0000", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.754e-06 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214629+0000", 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.990671623 2024-09-14T04:16:55.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214640+0000", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.049e-05 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379336+0000", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1646960749999999 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379484+0000", 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014773 2024-09-14T04:16:55.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379590+0000", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106259 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384020+0000", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044296270000000002 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384076+0000", 2024-09-14T04:16:55.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6487999999999999e-05 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390537+0000", 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064613240000000001 2024-09-14T04:16:55.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390591+0000", 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3164999999999997e-05 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.392963+0000", 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023726490000000001 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.758 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85211 10.2 10:452ef993:::benchmark_data_smithi001_190881_object85210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 426.456738927, 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1165732400000001, 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85211 2024-09-14T04:16:55.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278202+0000", 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9640000000000002e-06 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278222+0000", 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9796000000000002e-05 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278234+0000", 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1922999999999999e-05 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278242+0000", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3000000000000002e-06 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278320+0000", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8647999999999996e-05 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278333+0000", 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.257e-05 2024-09-14T04:16:55.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214669+0000", 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.936335849 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214677+0000", 2024-09-14T04:16:55.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1559999999999995e-06 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381729+0000", 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.167051839 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381867+0000", 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013809400000000001 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:16:55.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381978+0000", 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011068 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384104+0000", 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002125886 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.386784+0000", 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002680038 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.765 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393032+0000", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062480950000000004 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393073+0000", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1409999999999998e-05 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.394773+0000", 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001699992 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.766 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:157894 10.4 10:2bdef4ea:::benchmark_data_smithi001_190881_object42754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 328.35746574400002, 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.14587671099999999, 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.767 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 157894 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377474+0000", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7199999999999997e-07 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377479+0000", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1019999999999996e-06 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377491+0000", 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2379e-05 2024-09-14T04:16:55.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377498+0000", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.849e-06 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.408602+0000", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031104053 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.409004+0000", 2024-09-14T04:16:55.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040173900000000002 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.409491+0000", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048706299999999998 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.439679+0000", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030187535000000001 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.439712+0000", 2024-09-14T04:16:55.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3433000000000003e-05 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.448849+0000", 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0091365809999999995 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.518170+0000", 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.069321764999999994 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.518497+0000", 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032694199999999999 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.523350+0000", 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004852398 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:211990 10.4 10:28009dd3:::benchmark_data_smithi001_190881_object96850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 256.562034251, 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21457178399999999, 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 211990 2024-09-14T04:16:55.774 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:16:55.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5499999999999997e-07 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172909+0000", 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9349999999999996e-06 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172918+0000", 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.149e-06 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172925+0000", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4790000000000001e-06 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.177688+0000", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004763321 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.177879+0000", 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019074200000000001 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.178050+0000", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017125800000000001 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.182884+0000", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00483411 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.188759+0000", 2024-09-14T04:16:55.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058751059999999997 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220023+0000", 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031263460999999999 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.385148+0000", 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.16512544300000001 2024-09-14T04:16:55.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.385215+0000", 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6970000000000004e-05 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.387476+0000", 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022610550000000001 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.780 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212006 10.4 10:20e112b0:::benchmark_data_smithi001_190881_object96866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 256.54348819000001, 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21726297, 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.781 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212006 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191452+0000", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.728e-06 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191458+0000", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8810000000000001e-06 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191474+0000", 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5364999999999999e-05 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191482+0000", 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9650000000000008e-06 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191550+0000", 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8058999999999993e-05 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191713+0000", 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000163497 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191870+0000", 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015658499999999999 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.195822+0000", 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039520839999999998 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.198227+0000", 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002405084 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220065+0000", 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021838446000000001 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.403595+0000", 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18352981800000001 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.404044+0000", 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00044926699999999999 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408714+0000", 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0046691909999999996 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212011 10.4 10:2e1943d4:::benchmark_data_smithi001_190881_object96871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 256.53448627500001, 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21107719999999999, 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.788 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212011 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200454+0000", 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.587e-06 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200459+0000", 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4680000000000003e-06 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200471+0000", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2289e-05 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200476+0000", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4299999999999997e-06 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200535+0000", 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8785e-05 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200717+0000", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000181955 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200904+0000", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000187061 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208534+0000", 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007629767 2024-09-14T04:16:55.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208551+0000", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7577000000000001e-05 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220113+0000", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011561235 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408793+0000", 2024-09-14T04:16:55.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.188680184 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408872+0000", 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9589e-05 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411530+0000", 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026572729999999999 2024-09-14T04:16:55.794 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212013 10.4 10:2eef1eae:::benchmark_data_smithi001_190881_object96873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 256.53178688000003, 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21087093100000001, 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.795 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212013 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203154+0000", 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2019999999999998e-06 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203161+0000", 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203175+0000", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4576000000000001e-05 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203184+0000", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1259999999999998e-06 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204557+0000", 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001373785 2024-09-14T04:16:55.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204722+0000", 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000165119 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204910+0000", 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000187319 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208584+0000", 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003674053 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208599+0000", 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5474000000000001e-05 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220157+0000", 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011557507 2024-09-14T04:16:55.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411647+0000", 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.19149049900000001 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411710+0000", 2024-09-14T04:16:55.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2779000000000003e-05 2024-09-14T04:16:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414023+0000", 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023126840000000002 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.804 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212022 10.4 10:2ef778a0:::benchmark_data_smithi001_190881_object96882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 256.50572370399999, 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18721781100000001, 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212022 2024-09-14T04:16:55.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229217+0000", 2024-09-14T04:16:55.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8169999999999999e-06 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229223+0000", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4989999999999999e-06 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229240+0000", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6529999999999999e-05 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229247+0000", 2024-09-14T04:16:55.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1829999999999997e-06 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229299+0000", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2008000000000001e-05 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229414+0000", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011464 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229519+0000", 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105183 2024-09-14T04:16:55.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.232915+0000", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033962390000000001 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.233109+0000", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019340500000000001 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.237422+0000", 2024-09-14T04:16:55.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043136060000000002 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414077+0000", 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.17665492199999999 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414133+0000", 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5883e-05 2024-09-14T04:16:55.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.416433+0000", 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022998960000000001 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:26525 11.0 11:039722ec:::benchmark_data_smithi001_228075_object26524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 155.47451946000001, 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18390612100000001, 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.811 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 26525 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260421+0000", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3990000000000001e-06 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260444+0000", 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3757999999999999e-05 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260462+0000", 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7589999999999999e-05 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260470+0000", 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6329999999999996e-06 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260525+0000", 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5046000000000001e-05 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260653+0000", 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012794799999999999 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260882+0000", 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022941300000000001 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263376+0000", 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024943909999999999 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263962+0000", 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00058602100000000005 2024-09-14T04:16:55.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444206+0000", 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18024396200000001 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444271+0000", 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4189000000000005e-05 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444325+0000", 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4771000000000001e-05 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:26527 11.0 11:09da525d:::benchmark_data_smithi001_228075_object26526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 155.47420342500001, 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18375396499999999, 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.818 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 26527 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260736+0000", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7400000000000002e-07 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260763+0000", 2024-09-14T04:16:55.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7183999999999999e-05 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260777+0000", 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3246e-05 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260783+0000", 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6660000000000002e-06 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262177+0000", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001394045 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262318+0000", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014096900000000001 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262488+0000", 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016987799999999999 2024-09-14T04:16:55.822 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263718+0000", 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012298980000000001 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.265281+0000", 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001562995 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444390+0000", 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.17910886600000001 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444451+0000", 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0761000000000003e-05 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444489+0000", 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8583e-05 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.824 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111729 11.c 11:38344d7b:::benchmark_data_smithi001_228075_object111728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 83.337478891999993, 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0573153820000001, 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.825 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111729 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8099999999999998e-07 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397510+0000", 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0049999999999997e-05 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397517+0000", 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9469999999999997e-06 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.828 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397522+0000", 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9030000000000003e-06 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410401+0000", 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012879214 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.829 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410536+0000", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013432900000000001 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410682+0000", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014610799999999999 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420457+0000", 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097754369999999997 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432471+0000", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012013348 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438812+0000", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063412800000000004 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438822+0000", 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0463e-05 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454678+0000", 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0158558950000001 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454775+0000", 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7027000000000005e-05 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.832 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111734 11.c 11:3256a745:::benchmark_data_smithi001_228075_object111733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 83.330454304, 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0505815119999999, 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111734 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.833 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404486+0000", 2024-09-14T04:16:55.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.375e-06 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404540+0000", 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4305000000000003e-05 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404552+0000", 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2058999999999999e-05 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404560+0000", 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5070000000000003e-06 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420484+0000", 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015924223000000001 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420624+0000", 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000139889 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420770+0000", 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014612800000000001 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432477+0000", 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011706658999999999 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438869+0000", 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006392516 2024-09-14T04:16:55.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438876+0000", 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1420000000000004e-06 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454995+0000", 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161184889999999 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455066+0000", 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1219999999999999e-05 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111737 11.c 11:30a256cb:::benchmark_data_smithi001_228075_object111736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 83.329954009000005, 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.072352075, 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.840 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111737 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404986+0000", 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5600000000000004e-07 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405008+0000", 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2568000000000001e-05 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405023+0000", 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4442e-05 2024-09-14T04:16:55.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405028+0000", 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1789999999999999e-06 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427311+0000", 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022282885999999998 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427456+0000", 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014496299999999999 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.428041+0000", 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00058510400000000003 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438907+0000", 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010866310000000001 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438913+0000", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455100+0000", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161871069999999 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for degraded object", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455507+0000", 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040602299999999999 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477138+0000", 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021631583999999999 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477337+0000", 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000198717 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111746 11.c 11:38d82d32:::benchmark_data_smithi001_228075_object111745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:16:55.848 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 83.320868868000005, 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0634765070000001, 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111746 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:16:55.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414071+0000", 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.519e-06 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414094+0000", 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2546999999999999e-05 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414108+0000", 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4425e-05 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414117+0000", 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999992e-06 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438937+0000", 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024819529 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438942+0000", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1930000000000001e-06 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455547+0000", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016605078 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455715+0000", 2024-09-14T04:16:55.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001685 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455935+0000", 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000219936 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477381+0000", 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021446094999999998 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477395+0000", 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3925e-05 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477458+0000", 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2985999999999996e-05 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477546+0000", 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8023999999999996e-05 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.855 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111759 11.c 11:3f216db5:::benchmark_data_smithi001_228075_object111758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 83.292046337000002, 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0348463530000001, 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111759 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442895+0000", 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4980000000000001e-06 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442917+0000", 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2342999999999999e-05 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442932+0000", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4917e-05 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442940+0000", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.892e-06 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443001+0000", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1416999999999998e-05 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:16:55.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443012+0000", 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.092e-05 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459584+0000", 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0165711500000001 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459713+0000", 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012947499999999999 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459870+0000", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015698099999999999 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477555+0000", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017684921999999999 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477612+0000", 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7117000000000002e-05 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477662+0000", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9512000000000003e-05 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477739+0000", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7208999999999999e-05 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170311 11.0 11:0a21328e:::benchmark_data_smithi001_228075_object51996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.796022817000001, 2024-09-14T04:16:55.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.096979240999999994, 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170311 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938917+0000", 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.184e-06 2024-09-14T04:16:55.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938922+0000", 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9370000000000003e-06 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938934+0000", 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1569999999999999e-05 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938940+0000", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5939999999999999e-06 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938991+0000", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0785999999999998e-05 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939081+0000", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0138999999999999e-05 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:16:55.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939147+0000", 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6026999999999997e-05 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.945935+0000", 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0067877959999999996 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.959189+0000", 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013253699000000001 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035759+0000", 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.076570605999999999 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035846+0000", 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6504999999999999e-05 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035895+0000", 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9397999999999997e-05 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170312 11.0 11:088ab68c:::benchmark_data_smithi001_228075_object51997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.794980856999999, 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.103849861, 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.870 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.871 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.871 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.871 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.871 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170312 2024-09-14T04:16:55.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939959+0000", 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2399999999999996e-07 2024-09-14T04:16:55.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939962+0000", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0910000000000001e-06 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939973+0000", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1144999999999999e-05 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939980+0000", 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2779999999999996e-06 2024-09-14T04:16:55.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942366+0000", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023860040000000002 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942501+0000", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013510700000000001 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942608+0000", 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010642299999999999 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.946040+0000", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003432189 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.963603+0000", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017562782999999998 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043663+0000", 2024-09-14T04:16:55.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080059882999999998 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043758+0000", 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5027999999999999e-05 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043808+0000", 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0006000000000002e-05 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170322 11.0 11:03b3ce39:::benchmark_data_smithi001_228075_object52007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.778560450000001, 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.087605957999999998, 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170322 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956380+0000", 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2720000000000001e-06 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956384+0000", 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7319999999999998e-06 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956393+0000", 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.789e-06 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956398+0000", 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1590000000000001e-06 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956440+0000", 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1471000000000003e-05 2024-09-14T04:16:55.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956534+0000", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3986000000000004e-05 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956604+0000", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9989999999999999e-05 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.959203+0000", 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025993069999999999 2024-09-14T04:16:55.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.963657+0000", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044537209999999999 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043860+0000", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080202829000000003 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043943+0000", 2024-09-14T04:16:55.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3700000000000002e-05 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043984+0000", 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1001999999999998e-05 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:187727 11.f e271/260 11:fa5c2a43:::benchmark_data_smithi001_228075_object69412:head v 271'11856, mlcod=271'11853)", 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:55.080344+0000", 2024-09-14T04:16:55.884 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.65459511199999998, 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.062013575000000001, 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080344+0000", 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:16:55.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080344+0000", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080344+0000", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5499999999999997e-07 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080350+0000", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4960000000000004e-06 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:16:55.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080359+0000", 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8729999999999996e-06 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080365+0000", 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0020000000000003e-06 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080426+0000", 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1490999999999998e-05 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080445+0000", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8529000000000002e-05 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142286+0000", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.061841127000000003 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142357+0000", 2024-09-14T04:16:55.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1302000000000003e-05 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:16:55.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:55.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:55.890 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:55 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:16:55.890 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1593915581' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:16:55.890 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:55 smithi012 ceph-mon[101180]: osdmap e271: 8 total, 7 up, 8 in 2024-09-14T04:16:55.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:56.025 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:56.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:56.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:56.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:56.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:56.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:56.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:56.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:56.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:56.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:56.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:56.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:56 smithi001 ceph-mon[107399]: pgmap v972: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 20767/156354 objects degraded (13.282%); 8.4 MiB/s, 159 objects/s recovering 2024-09-14T04:16:56.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:56 smithi001 ceph-mon[112165]: pgmap v972: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 20767/156354 objects degraded (13.282%); 8.4 MiB/s, 159 objects/s recovering 2024-09-14T04:16:56.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:56.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:56 smithi012 ceph-mon[101180]: pgmap v972: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 20767/156354 objects degraded (13.282%); 8.4 MiB/s, 159 objects/s recovering 2024-09-14T04:16:56.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:56.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:56.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:16:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:16:56] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-09-14T04:16:56.998 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:57.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:57.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:57.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:57.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:57.294 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:57.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:57.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:57.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:57.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:57.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:57.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:57.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:57.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:57.849 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2024-09-14T04:16:57.849 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:16:57.849 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-14T04:16:57.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-14T04:16:57.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:16:57.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:57.979 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:57.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:58.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:58.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:58.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:58.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:58.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:58.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:16:58.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:16:58.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:58.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:58.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:58.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:58.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:58 smithi001 ceph-mon[107399]: pgmap v973: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19108/143199 objects degraded (13.344%); 7.1 MiB/s, 157 objects/s recovering 2024-09-14T04:16:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:58 smithi001 ceph-mon[112165]: pgmap v973: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19108/143199 objects degraded (13.344%); 7.1 MiB/s, 157 objects/s recovering 2024-09-14T04:16:58.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:58.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:58 smithi012 ceph-mon[101180]: pgmap v973: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19108/143199 objects degraded (13.344%); 7.1 MiB/s, 157 objects/s recovering 2024-09-14T04:16:58.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:16:58.975 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:58.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:16:59.077 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:59.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:16:59.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:16:59.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:16:59.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:16:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.402 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:59.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:16:59.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:16:59.544 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:16:59.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:59.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:59.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:16:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 19108/143199 objects degraded (13.344%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:59.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:16:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 19108/143199 objects degraded (13.344%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:59.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:16:59.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:16:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:16:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:16:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 19108/143199 objects degraded (13.344%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:16:59.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:16:59.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:16:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:16:59.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:00.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:00.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:00.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:00.242 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:00.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:17:00.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:17:00.340 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:17:00.340 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:17:00.342 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:00.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:00.381 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:17:00.381 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:17:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:00.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:00.420 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:17:00.421 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:17:00.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:17:00.457 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:17:00.458 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:17:00.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:00.493 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:17:00.493 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:17:00.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:00.536 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:17:00.537 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:17:00.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:00 smithi001 ceph-mon[107399]: pgmap v974: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 18383/142329 objects degraded (12.916%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:00.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3318840791' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:00.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:00 smithi001 ceph-mon[112165]: pgmap v974: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 18383/142329 objects degraded (12.916%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:00.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3318840791' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:00.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:00.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:00.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:00.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:00.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:00 smithi012 ceph-mon[101180]: pgmap v974: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 18383/142329 objects degraded (12.916%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3318840791' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:00.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:00.925 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:00.984 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:00.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:01.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:01.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:01.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:01.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.403 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:01.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:01.522 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:17:01.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:01.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:01.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:01.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:01.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:01.959 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:01.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:02.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:02.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:02.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:02.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.227 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:02.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:02.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:02.354 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:02.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:02.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:02.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:02.628 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:02 smithi001 ceph-mon[107399]: pgmap v975: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 17831/138924 objects degraded (12.835%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:02 smithi001 ceph-mon[112165]: pgmap v975: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 17831/138924 objects degraded (12.835%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:02.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:02.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:02 smithi012 ceph-mon[101180]: pgmap v975: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 17831/138924 objects degraded (12.835%); 6.9 MiB/s, 153 objects/s recovering 2024-09-14T04:17:02.874 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:02.913 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:02.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:03.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:03.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:03.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:03.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:03.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:03.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:03.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:03.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:03.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:03.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:17:03.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:03.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:03.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:03.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:03.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:03.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:03.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:03.962 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:04.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:04.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:04.044 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:04.048 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:04.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:04.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:04.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:04.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:04.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:04.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:04 smithi001 ceph-mon[112165]: pgmap v976: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 16712/129012 objects degraded (12.954%); 6.9 MiB/s, 155 objects/s recovering 2024-09-14T04:17:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 16712/129012 objects degraded (12.954%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:04 smithi001 ceph-mon[107399]: pgmap v976: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 16712/129012 objects degraded (12.954%); 6.9 MiB/s, 155 objects/s recovering 2024-09-14T04:17:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 16712/129012 objects degraded (12.954%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:04.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:04.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:04.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.714 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:04.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:04.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:04 smithi012 ceph-mon[101180]: pgmap v976: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 16712/129012 objects degraded (12.954%); 6.9 MiB/s, 155 objects/s recovering 2024-09-14T04:17:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 16712/129012 objects degraded (12.954%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:04.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:04.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:04.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:04.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:05.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:05.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:05.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:05.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:05.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:05.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:05.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:05.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:05.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:05.692 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:05.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:05.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:05.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:05.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:05.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:05.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:06.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:06.108 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:06.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:06.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[107399]: pgmap v977: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 15974/128187 objects degraded (12.461%); 7.0 MiB/s, 153 objects/s recovering 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:05 smithi001 ceph-mon[112165]: pgmap v977: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 15974/128187 objects degraded (12.461%); 7.0 MiB/s, 153 objects/s recovering 2024-09-14T04:17:06.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:05 smithi012 ceph-mon[101180]: pgmap v977: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 15974/128187 objects degraded (12.461%); 7.0 MiB/s, 153 objects/s recovering 2024-09-14T04:17:06.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:06.403 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:06.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:06.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:06.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:06.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:06.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:06.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:06.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:06.911 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:06.911 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:06.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:06] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-09-14T04:17:06.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:06.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:07.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:07.117 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:07.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.254 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:07.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:07.403 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:07.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:07.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:07.620 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:17:07.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:07.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:07.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:07.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:07.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:07.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:08.065 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:08.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:08.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:08.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:08.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:08.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:08.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:08.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:08.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:08 smithi001 ceph-mon[107399]: pgmap v978: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 14320/115428 objects degraded (12.406%); 6.3 MiB/s, 160 objects/s recovering 2024-09-14T04:17:08.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:08 smithi001 ceph-mon[112165]: pgmap v978: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 14320/115428 objects degraded (12.406%); 6.3 MiB/s, 160 objects/s recovering 2024-09-14T04:17:08.505 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:08.533 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:08.601 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:08 smithi012 ceph-mon[101180]: pgmap v978: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 14320/115428 objects degraded (12.406%); 6.3 MiB/s, 160 objects/s recovering 2024-09-14T04:17:08.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:08.661 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:08.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:08.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:08.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:08.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:08.914 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:08.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:09.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:09.039 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:09.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:09.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:09.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:09.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:09.333 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:17:09.333 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:17:09.333 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:17:09.333 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:17:09.333 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:17:09.334 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:17:09.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:09.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:09.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:09.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:09.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:09.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:09 smithi001 ceph-mon[107399]: osdmap e272: 8 total, 7 up, 8 in 2024-09-14T04:17:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 14320/115428 objects degraded (12.406%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:09 smithi001 ceph-mon[112165]: osdmap e272: 8 total, 7 up, 8 in 2024-09-14T04:17:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 14320/115428 objects degraded (12.406%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:09.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:09 smithi012 ceph-mon[101180]: osdmap e272: 8 total, 7 up, 8 in 2024-09-14T04:17:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 14320/115428 objects degraded (12.406%), 10 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:09.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:09.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:09.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:10.004 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:10.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:10.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:10.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:10.280 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:10.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:10.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:10.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:10.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:10.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:10.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:10 smithi001 ceph-mon[107399]: pgmap v980: 99 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 13850/114609 objects degraded (12.085%); 6.8 MiB/s, 158 objects/s recovering 2024-09-14T04:17:10.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:10 smithi001 ceph-mon[107399]: osdmap e273: 8 total, 7 up, 8 in 2024-09-14T04:17:10.665 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:10 smithi001 ceph-mon[112165]: pgmap v980: 99 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 13850/114609 objects degraded (12.085%); 6.8 MiB/s, 158 objects/s recovering 2024-09-14T04:17:10.666 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:10 smithi001 ceph-mon[112165]: osdmap e273: 8 total, 7 up, 8 in 2024-09-14T04:17:10.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:10.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:10.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:10 smithi012 ceph-mon[101180]: pgmap v980: 99 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 13850/114609 objects degraded (12.085%); 6.8 MiB/s, 158 objects/s recovering 2024-09-14T04:17:10.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:10 smithi012 ceph-mon[101180]: osdmap e273: 8 total, 7 up, 8 in 2024-09-14T04:17:10.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:10.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:10.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:11.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:11.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:11.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:11.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:11.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:11.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:11.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:17:11.413 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:17:11.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-14T04:17:11.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:11.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:11.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:11.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:11.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:11.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:11.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:11.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:11.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:12.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:12.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:12.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:12.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:12.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:12.345 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:12.436 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:12.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:12.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:12.578 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[112165]: pgmap v982: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 13369/111825 objects degraded (11.955%); 8.5 MiB/s, 167 objects/s recovering 2024-09-14T04:17:12.579 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.579 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.579 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[107399]: pgmap v982: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 13369/111825 objects degraded (11.955%); 8.5 MiB/s, 167 objects/s recovering 2024-09-14T04:17:12.579 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.579 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:12.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:12.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:12.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.761 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:12.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:12 smithi012 ceph-mon[101180]: pgmap v982: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 13369/111825 objects degraded (11.955%); 8.5 MiB/s, 167 objects/s recovering 2024-09-14T04:17:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:12.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:12.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:12.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:12.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:13.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:13.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:13.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:13.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:13.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:13.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:13.449 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:13.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:13.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:13.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:13.685 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:13.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:13.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:13.861 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:13.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:13.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:14.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:14.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:14.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:14.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:14.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:14.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:14.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:14.365 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:17:14.366 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:17:14.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:14.399 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:17:14.399 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:17:14.400 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:14.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:14.438 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:17:14.439 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:17:14.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:17:14.473 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:17:14.507 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:17:14.507 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:17:14.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:14.540 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:17:14.541 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:17:14.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:14.568 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:17:14.568 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[107399]: pgmap v983: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12207/103713 objects degraded (11.770%); 3.4 MiB/s, 138 objects/s recovering 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12207/103713 objects degraded (11.770%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3821478894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[112165]: pgmap v983: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12207/103713 objects degraded (11.770%); 3.4 MiB/s, 138 objects/s recovering 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12207/103713 objects degraded (11.770%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:14.569 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:17:14.570 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3821478894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:14.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.692 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:14.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:14.699 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:17:14.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:14.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:14 smithi012 ceph-mon[101180]: pgmap v983: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 68 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12207/103713 objects degraded (11.770%); 3.4 MiB/s, 138 objects/s recovering 2024-09-14T04:17:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12207/103713 objects degraded (11.770%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:17:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3821478894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:14.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:14.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:14.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:15.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:15.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:15.239 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.280 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:15.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:15.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:15.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:15.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:15.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:15.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:15.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:15.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:17:15.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[107399]: osdmap e274: 8 total, 7 up, 8 in 2024-09-14T04:17:15.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:15.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:17:15.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:15 smithi001 ceph-mon[112165]: osdmap e274: 8 total, 7 up, 8 in 2024-09-14T04:17:15.698 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:15.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:15.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:15 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1008965922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:17:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:15 smithi012 ceph-mon[101180]: osdmap e274: 8 total, 7 up, 8 in 2024-09-14T04:17:15.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:15.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:15.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:15.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:16.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:16.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:16.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:16.249 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:16.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:16.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:16.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:16.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:16.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:16.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:16.680 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:16 smithi001 ceph-mon[107399]: pgmap v985: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12207/102807 objects degraded (11.874%); 4.0 MiB/s, 122 objects/s recovering 2024-09-14T04:17:16.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:16 smithi001 ceph-mon[112165]: pgmap v985: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12207/102807 objects degraded (11.874%); 4.0 MiB/s, 122 objects/s recovering 2024-09-14T04:17:16.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:16.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:16 smithi012 ceph-mon[101180]: pgmap v985: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 12207/102807 objects degraded (11.874%); 4.0 MiB/s, 122 objects/s recovering 2024-09-14T04:17:16.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:16.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:16.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:16] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-09-14T04:17:17.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:17.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:17.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.220 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:17.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:17.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:17.362 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:17.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:17.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:17.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:17.640 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:17.741 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:17.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:17.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:17.906 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:17.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:17.976 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:17.977 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:17:18.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:18.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:18.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:18.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:18.223 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:17:18.223 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:17:18.223 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:17:18.223 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:17:18.223 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:17:18.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:18.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:18.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.469 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:18.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:18.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:18.600 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:18.701 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:18.713 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:18 smithi012 ceph-mon[101180]: pgmap v986: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10388/89076 objects degraded (11.662%); 0 B/s, 88 objects/s recovering 2024-09-14T04:17:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:18 smithi001 ceph-mon[107399]: pgmap v986: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10388/89076 objects degraded (11.662%); 0 B/s, 88 objects/s recovering 2024-09-14T04:17:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:18 smithi001 ceph-mon[112165]: pgmap v986: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 10388/89076 objects degraded (11.662%); 0 B/s, 88 objects/s recovering 2024-09-14T04:17:18.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:18.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:18.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:18.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:18.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:19.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:19.108 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:19.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:19.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:19.278 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:19.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:17:19.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:19.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:19.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:19.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:19.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10388/89076 objects degraded (11.662%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10388/89076 objects degraded (11.662%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:19.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:19.835 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10388/89076 objects degraded (11.662%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:19.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:19.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:19.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:17:19.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:19.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:20.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:20.135 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:20.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:20.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:20.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:20.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:20.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:20.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[107399]: pgmap v987: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10388/88194 objects degraded (11.779%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[112165]: pgmap v987: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10388/88194 objects degraded (11.779%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:20.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:20.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:20.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:20.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:20.737 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:20.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:20 smithi012 ceph-mon[101180]: pgmap v987: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10388/88194 objects degraded (11.779%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:17:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:20.849 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:20.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:20.951 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:20.993 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:20.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:21.094 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:21.125 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:21.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:21.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.269 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:21.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:21.371 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:21.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:21.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:21.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:21.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:21.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:21.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:21.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:21.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:21.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:17:21.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:21.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:21.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:22.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:22.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:22.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:22.251 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:22.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:22.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:22.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:22.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:22.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:22.609 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:22 smithi001 ceph-mon[107399]: pgmap v988: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 9733/84324 objects degraded (11.542%); 0 B/s, 70 objects/s recovering 2024-09-14T04:17:22.609 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:22 smithi001 ceph-mon[112165]: pgmap v988: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 9733/84324 objects degraded (11.542%); 0 B/s, 70 objects/s recovering 2024-09-14T04:17:22.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:22.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:22.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:22.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:22 smithi012 ceph-mon[101180]: pgmap v988: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 9733/84324 objects degraded (11.542%); 0 B/s, 70 objects/s recovering 2024-09-14T04:17:22.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:22.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:22.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:23.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:23.098 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:23.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:23.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:23.310 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:23.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:23.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:23.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:23.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:23.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:23.653 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:23.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:23.803 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:23.905 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:23.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:23.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:24.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:24.074 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:24.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:24.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:24.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:24.364 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:24.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:24.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:24 smithi001 ceph-mon[107399]: pgmap v989: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 8495/74409 objects degraded (11.417%); 0 B/s, 61 objects/s recovering 2024-09-14T04:17:24.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8495/74409 objects degraded (11.417%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:24 smithi001 ceph-mon[112165]: pgmap v989: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 8495/74409 objects degraded (11.417%); 0 B/s, 61 objects/s recovering 2024-09-14T04:17:24.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8495/74409 objects degraded (11.417%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:24.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:24.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:24.636 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:24.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:24.775 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:24.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:24 smithi012 ceph-mon[101180]: pgmap v989: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 8495/74409 objects degraded (11.417%); 0 B/s, 61 objects/s recovering 2024-09-14T04:17:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8495/74409 objects degraded (11.417%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:24.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:24.922 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:24.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:25.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:25.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:25.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:25.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:25.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:25.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:25.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:25.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:25.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:25.609 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:25.711 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:25.741 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:25.781 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:17:25.781 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:17:25.781 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:17:25.781 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:17:25.781 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:17:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:17:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:17:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:17:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:17:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:17:25.783 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:17:25.783 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:17:25.843 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:25.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:25.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:25.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:26.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:26.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:26.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:26.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:26.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:26.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:26.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:26.409 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:17:26.409 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:17:26.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:26.439 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:17:26.439 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:17:26.439 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:26.468 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:17:26.468 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:17:26.494 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:17:26.494 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:211983 10.d e233/210 10:bd78eaaa:::benchmark_data_smithi001_190881_object96843:head v 233'13216, mlcod=233'13211)", 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.171689+0000", 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 287.09591692100003, 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22182081200000001, 2024-09-14T04:17:26.519 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171689+0000", 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171689+0000", 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171690+0000", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5600000000000004e-07 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171696+0000", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3339999999999998e-06 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171707+0000", 2024-09-14T04:17:26.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0716e-05 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171713+0000", 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6000000000000003e-06 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171759+0000", 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5883000000000001e-05 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.171780+0000", 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1191000000000001e-05 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.393453+0000", 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22167245299999999 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.393510+0000", 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6779e-05 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.523 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45035.0:211994 10.d e233/210 10:bee466df:::benchmark_data_smithi001_190881_object96854:head v 233'13218, mlcod=233'13212)", 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.181572+0000", 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 287.08603414700002, 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21980951100000001, 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.524 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181572+0000", 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181572+0000", 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181573+0000", 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8200000000000008e-07 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181579+0000", 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.297e-06 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181592+0000", 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2646999999999999e-05 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.526 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.181599+0000", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7500000000000003e-06 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.182153+0000", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055333800000000001 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.182222+0000", 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9269000000000004e-05 2024-09-14T04:17:26.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.401223+0000", 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21900124800000001 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.401381+0000", 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015798 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111739 11.2 11:4a5bd84c:::benchmark_data_smithi001_228075_object111738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 113.837664441, 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.00444102, 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.529 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111739 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429943+0000", 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.609e-06 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429968+0000", 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5333e-05 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429981+0000", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2408999999999999e-05 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429989+0000", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3669999999999997e-06 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430098+0000", 2024-09-14T04:17:26.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010843199999999999 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430104+0000", 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6599999999999998e-06 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.435778+0000", 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056739889999999999 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:17:26.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.435786+0000", 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.542e-06 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.431760+0000", 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99597435499999998 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.434382+0000", 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026223240000000001 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111747 11.2 11:41cf9340:::benchmark_data_smithi001_228075_object111746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 113.837596314, 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.005084791, 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111747 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430010+0000", 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6899999999999997e-07 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430032+0000", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1472e-05 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430039+0000", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9449999999999993e-06 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430045+0000", 2024-09-14T04:17:26.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1030000000000001e-06 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431434+0000", 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001389415 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431439+0000", 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2039999999999996e-06 2024-09-14T04:17:26.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436670+0000", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052304860000000003 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436679+0000", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9269999999999994e-06 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.434799+0000", 2024-09-14T04:17:26.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99812029800000002 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.435094+0000", 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00029537200000000002 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111753 11.2 11:4fdf47a5:::benchmark_data_smithi001_228075_object111752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:17:26.540 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 113.83520471, 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.017779915, 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111753 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.541 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432403+0000", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4219999999999999e-06 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432424+0000", 2024-09-14T04:17:26.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1897999999999999e-05 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432480+0000", 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5624999999999997e-05 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432488+0000", 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.038e-06 2024-09-14T04:17:26.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436702+0000", 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042138000000000002 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436707+0000", 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9250000000000003e-06 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.435120+0000", 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99841299800000005 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.436382+0000", 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012620559999999999 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.437292+0000", 2024-09-14T04:17:26.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00091012899999999997 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447762+0000", 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010469882999999999 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.449982+0000", 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022205049999999998 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.450085+0000", 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010273200000000001 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.450181+0000", 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5903999999999995e-05 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.548 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:170312 11.0 e265/244 11:088ab68c:::benchmark_data_smithi001_228075_object51997:head v 265'10803, mlcod=265'10799)", 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.942679+0000", 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.324926310000002, 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.100691406, 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942679+0000", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942679+0000", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942680+0000", 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7599999999999996e-07 2024-09-14T04:17:26.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942685+0000", 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5449999999999997e-06 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942695+0000", 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9510000000000001e-06 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942701+0000", 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2469999999999997e-06 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.025898+0000", 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.083196669000000001 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.025920+0000", 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2558e-05 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043330+0000", 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017410101000000001 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043371+0000", 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0459e-05 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170319 11.9 11:9584ec65:::benchmark_data_smithi001_228075_object52004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.948905+0000", 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 52.318700905, 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.102377093, 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.554 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170319 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.948905+0000", 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.948905+0000", 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.948906+0000", 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3289999999999999e-06 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.948911+0000", 2024-09-14T04:17:26.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5480000000000003e-06 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.024610+0000", 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.075699196999999996 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.024624+0000", 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.363e-05 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.039878+0000", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015254696999999999 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.040033+0000", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015477899999999999 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.040144+0000", 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011134200000000001 2024-09-14T04:17:26.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.047465+0000", 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073203319999999997 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.047506+0000", 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1606000000000001e-05 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.051153+0000", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003647094 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.051227+0000", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3639000000000004e-05 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.051282+0000", 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.49e-05 2024-09-14T04:17:26.560 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:182787 11.9 11:9f2473ea:::benchmark_data_smithi001_228075_object64472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:49.880228+0000", 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 36.387377542999999, 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041591503000000002, 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.561 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182787 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880228+0000", 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880228+0000", 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880229+0000", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3099999999999997e-07 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880232+0000", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3220000000000001e-06 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880240+0000", 2024-09-14T04:17:26.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2370000000000003e-06 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880246+0000", 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9789999999999998e-06 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.883778+0000", 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035328730000000002 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.883904+0000", 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012557200000000001 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.884020+0000", 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115696 2024-09-14T04:17:26.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.891496+0000", 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0074764590000000004 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.902144+0000", 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010647792 2024-09-14T04:17:26.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921565+0000", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019421087 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921779+0000", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021398500000000001 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921820+0000", 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0769999999999998e-05 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.567 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:182788 11.9 11:9c59d4f4:::benchmark_data_smithi001_228075_object64473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:49.880398+0000", 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 36.387207377999999, 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041580130999999999, 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.568 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182788 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880398+0000", 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880398+0000", 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880399+0000", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6400000000000002e-07 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880403+0000", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7000000000000002e-06 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880413+0000", 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0519999999999999e-05 2024-09-14T04:17:26.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.880420+0000", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9879999999999998e-06 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.887232+0000", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0068114150000000004 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.887333+0000", 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101408 2024-09-14T04:17:26.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.887439+0000", 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105431 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.891553+0000", 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041146849999999999 2024-09-14T04:17:26.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.902222+0000", 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010668265 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921837+0000", 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019615815000000002 2024-09-14T04:17:26.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921932+0000", 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4617999999999994e-05 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:49.921979+0000", 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6622000000000002e-05 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:186832 11.9 11:9afa6e04:::benchmark_data_smithi001_228075_object68517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:54.117846+0000", 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.149760153000003, 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.058314286, 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 186832 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117846+0000", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117846+0000", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117846+0000", 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1200000000000003e-07 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117849+0000", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6369999999999999e-06 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117855+0000", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6630000000000004e-06 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.117861+0000", 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.096e-06 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.124013+0000", 2024-09-14T04:17:26.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006152505 2024-09-14T04:17:26.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.124358+0000", 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034516499999999998 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.124490+0000", 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131851 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.137970+0000", 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013480011 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.139535+0000", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015645279999999999 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176055+0000", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.036520678000000001 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176114+0000", 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8211000000000002e-05 2024-09-14T04:17:26.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176160+0000", 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6428999999999999e-05 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:186842 11.9 11:9cbbf4ae:::benchmark_data_smithi001_228075_object68527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:54.132974+0000", 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.134631560000003, 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043292416, 2024-09-14T04:17:26.583 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 186842 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.132974+0000", 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.132974+0000", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.132976+0000", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8139999999999999e-06 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.132982+0000", 2024-09-14T04:17:26.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8479999999999997e-06 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.133000+0000", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7808999999999998e-05 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.133008+0000", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2349999999999999e-06 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.133178+0000", 2024-09-14T04:17:26.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016977499999999999 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.133495+0000", 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000317585 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.133923+0000", 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00042719600000000002 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.144756+0000", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010833453 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.145407+0000", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00065150600000000005 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176196+0000", 2024-09-14T04:17:26.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030788056000000001 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176236+0000", 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0519999999999998e-05 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:54.176267+0000", 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0618999999999999e-05 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.589 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:187732 11.b 11:d43cf343:::benchmark_data_smithi001_228075_object69417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:55.080418+0000", 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 31.187187625, 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.052710699, 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 187732 2024-09-14T04:17:26.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080418+0000", 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080418+0000", 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080420+0000", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.477e-06 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080424+0000", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.454e-06 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080436+0000", 2024-09-14T04:17:26.592 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1673e-05 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080442+0000", 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6810000000000001e-06 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080490+0000", 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7385000000000001e-05 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080589+0000", 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9319000000000002e-05 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.080693+0000", 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010351700000000001 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.084258+0000", 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035649039999999998 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.594 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.084483+0000", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022589299999999999 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.133035+0000", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.048551204000000001 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.133089+0000", 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4645000000000003e-05 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.133129+0000", 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9546999999999999e-05 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:187742 11.9 11:9e3123f9:::benchmark_data_smithi001_228075_object69427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:55.091976+0000", 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 31.175629574999999, 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.050845697000000002, 2024-09-14T04:17:26.596 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 187742 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091976+0000", 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.597 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091976+0000", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091978+0000", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.013e-06 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091983+0000", 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7110000000000003e-06 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091993+0000", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8670000000000006e-06 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.091998+0000", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4729999999999999e-06 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.092041+0000", 2024-09-14T04:17:26.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2407999999999998e-05 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.092115+0000", 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3949000000000006e-05 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.120035+0000", 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.027920240999999998 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.122805+0000", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002770392 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142478+0000", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019672965000000001 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142712+0000", 2024-09-14T04:17:26.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00023342299999999999 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142770+0000", 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8629999999999999e-05 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.142822+0000", 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1625000000000001e-05 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:187745 11.b 11:d85ca871:::benchmark_data_smithi001_228075_object69430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:55.097817+0000", 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 31.169788635, 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.047440804000000003, 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 187745 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.603 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.097817+0000", 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.097817+0000", 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.604 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.097818+0000", 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0100000000000004e-07 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.097822+0000", 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7890000000000001e-06 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.119951+0000", 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022129722000000001 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.119963+0000", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1104000000000001e-05 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.120009+0000", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6776000000000001e-05 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.120095+0000", 2024-09-14T04:17:26.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5438000000000004e-05 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.120179+0000", 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4029000000000005e-05 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.122892+0000", 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027129839999999999 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.145148+0000", 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022255789000000002 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.145187+0000", 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8998000000000002e-05 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.145234+0000", 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7494999999999998e-05 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:55.145258+0000", 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3879e-05 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:189860 11.b 11:d1c98dfe:::benchmark_data_smithi001_228075_object71545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:57.374095+0000", 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 28.893510485, 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041388714, 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.609 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 189860 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374095+0000", 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.610 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374095+0000", 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374097+0000", 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4720000000000001e-06 2024-09-14T04:17:26.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374103+0000", 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.37e-06 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374121+0000", 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8284e-05 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374129+0000", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9489999999999996e-06 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374190+0000", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0268e-05 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.374355+0000", 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016580699999999999 2024-09-14T04:17:26.613 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375010+0000", 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00065474499999999998 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.388846+0000", 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013835583 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.614 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.388858+0000", 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2694000000000001e-05 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.415377+0000", 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.026518672 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.415443+0000", 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5709000000000004e-05 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.415484+0000", 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1161000000000001e-05 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:189864 11.b 11:de6dba82:::benchmark_data_smithi001_228075_object71549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:57.375015+0000", 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 28.892590800000001, 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.045466911999999998, 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.616 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 189864 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375015+0000", 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375015+0000", 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375015+0000", 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4999999999999998e-07 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375018+0000", 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.689e-06 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375025+0000", 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5409999999999997e-06 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.375030+0000", 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4580000000000001e-06 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.379041+0000", 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0040106279999999996 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.379173+0000", 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013232900000000001 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.379308+0000", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013473900000000001 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.388901+0000", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0095931750000000007 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.408122+0000", 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019220636999999999 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.420250+0000", 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012128132 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.420407+0000", 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015743300000000001 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.622 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:57.420482+0000", 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4701000000000002e-05 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:212646 11.8 e274/244 11:1d812ef7:::benchmark_data_smithi001_228075_object94331:head v 274'13430, mlcod=274'13428)", 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:17:22.232542+0000", 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.0350635959999996, 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041780902000000002, 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.623 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232542+0000", 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232542+0000", 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232543+0000", 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.206e-06 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232552+0000", 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6680000000000007e-06 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232570+0000", 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8388000000000001e-05 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.232580+0000", 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0690000000000008e-06 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.234049+0000", 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001469822 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.234109+0000", 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9632000000000001e-05 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.274287+0000", 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.040177833000000003 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.274323+0000", 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6284e-05 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.35451.0:212651 11.0 e274/244 11:09ad9fa4:::benchmark_data_smithi001_228075_object94336:head v 274'13414, mlcod=274'13411)", 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:17:22.237688+0000", 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.0299179399999998, 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043181107000000003, 2024-09-14T04:17:26.628 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237688+0000", 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237688+0000", 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237689+0000", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.141e-06 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237696+0000", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5819999999999998e-06 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237710+0000", 2024-09-14T04:17:26.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.437e-05 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.237717+0000", 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2690000000000002e-06 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.243102+0000", 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053847280000000001 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.243120+0000", 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8291000000000002e-05 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.280823+0000", 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.037702685999999999 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.280869+0000", 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.604e-05 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:212857 11.9 11:999f1378:::benchmark_data_smithi001_228075_object94542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:17:22.460147+0000", 2024-09-14T04:17:26.633 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 3.807459218, 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.048259674000000002, 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212857 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460147+0000", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460147+0000", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460148+0000", 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0109999999999999e-06 2024-09-14T04:17:26.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460152+0000", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0269999999999997e-06 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460162+0000", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0199e-05 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.460168+0000", 2024-09-14T04:17:26.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5220000000000004e-06 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.499606+0000", 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.039437479999999997 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.500004+0000", 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00039819200000000001 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.500128+0000", 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000124237 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.508120+0000", 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079920560000000009 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.508182+0000", 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1221000000000002e-05 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.508266+0000", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4648999999999995e-05 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.508360+0000", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4314999999999994e-05 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:22.508406+0000", 2024-09-14T04:17:26.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5765e-05 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:215739 11.9 11:9a93a891:::benchmark_data_smithi001_228075_object97424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:17:25.374488+0000", 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.89311800699999999, 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041620894999999998, 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:17:26.640 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 215739 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374488+0000", 2024-09-14T04:17:26.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374488+0000", 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374489+0000", 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8800000000000001e-07 2024-09-14T04:17:26.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374494+0000", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8740000000000003e-06 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374508+0000", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4116e-05 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.374516+0000", 2024-09-14T04:17:26.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9969999999999998e-06 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.379610+0000", 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0050946760000000002 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.379713+0000", 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010238099999999999 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:17:26.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.383141+0000", 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034284459999999999 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.390266+0000", 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0071250740000000003 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.392928+0000", 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026618760000000001 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.412341+0000", 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019412834 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:17:26.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.415940+0000", 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035992009999999998 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:25.416109+0000", 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016853199999999999 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:17:26.647 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:17:26.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:26.649 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:17:26.649 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:17:26.649 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:17:26.650 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:17:26.650 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:17:26.650 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:17:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:26.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:26 smithi001 ceph-mon[112165]: pgmap v990: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 8495/73497 objects degraded (11.558%); 0 B/s, 57 objects/s recovering 2024-09-14T04:17:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3318954484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:26.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:26 smithi001 ceph-mon[107399]: pgmap v990: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 8495/73497 objects degraded (11.558%); 0 B/s, 57 objects/s recovering 2024-09-14T04:17:26.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3318954484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:26.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:26.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:26 smithi012 ceph-mon[101180]: pgmap v990: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 8495/73497 objects degraded (11.558%); 0 B/s, 57 objects/s recovering 2024-09-14T04:17:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3318954484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:26.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:26.958 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:26.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:26.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:26] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.51.0" 2024-09-14T04:17:27.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:27.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:27.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:27.228 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:27.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:27.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:27.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:27.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.507 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:27.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:27.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:27.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:27.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:27.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:27.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:27.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:27.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:28.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:28.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:28.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:28.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:28.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:28.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:28.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:28.467 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:28.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:28.604 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:28.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:28.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:28 smithi001 ceph-mon[107399]: pgmap v991: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 6620/59613 objects degraded (11.105%); 0 B/s, 79 objects/s recovering 2024-09-14T04:17:28.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:28 smithi001 ceph-mon[112165]: pgmap v991: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 6620/59613 objects degraded (11.105%); 0 B/s, 79 objects/s recovering 2024-09-14T04:17:28.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:28.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:28.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:28 smithi012 ceph-mon[101180]: pgmap v991: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 6620/59613 objects degraded (11.105%); 0 B/s, 79 objects/s recovering 2024-09-14T04:17:28.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:28.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:28.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:29.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:29.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:29.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:29.259 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:29.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:29.399 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:29.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:29.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:29.570 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:29.672 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:29.706 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:29.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6620/59613 objects degraded (11.105%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6620/59613 objects degraded (11.105%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:29.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:29.834 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:29.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6620/59613 objects degraded (11.105%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:29.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:29.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:29.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:30.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:30.118 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:30.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:30.264 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:30.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:17:30.378 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:17:30.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-14T04:17:30.409 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:30.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:30.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:30.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:30.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:30 smithi001 ceph-mon[107399]: pgmap v992: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 6620/58608 objects degraded (11.295%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:30 smithi001 ceph-mon[112165]: pgmap v992: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 6620/58608 objects degraded (11.295%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:30.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:30.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:30 smithi012 ceph-mon[101180]: pgmap v992: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 6620/58608 objects degraded (11.295%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:30.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:30.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:17:30.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:30.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:31.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:31.116 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:31.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:31.272 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:31.351 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:31.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:31.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:31.556 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:31.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:31.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:31.712 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:31.712 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:17:31.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:31.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:31.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:31.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:31.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:32.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:32.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:32.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:32.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:32.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:32.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:32.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:32.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:32.529 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:32.577 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:32.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:32.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:32.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:32 smithi001 ceph-mon[107399]: pgmap v993: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 5732/52662 objects degraded (10.885%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:32.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:32 smithi001 ceph-mon[112165]: pgmap v993: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 5732/52662 objects degraded (10.885%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:32.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:32.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:32 smithi012 ceph-mon[101180]: pgmap v993: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 5732/52662 objects degraded (10.885%); 0 B/s, 54 objects/s recovering 2024-09-14T04:17:32.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:32.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:32.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:33.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:33.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:33.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:33.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:33.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:33.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:33.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:33.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:33.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.561 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:33.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:33 smithi012 ceph-mon[101180]: osdmap e275: 8 total, 7 up, 8 in 2024-09-14T04:17:33.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:33.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:33.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:33 smithi001 ceph-mon[112165]: osdmap e275: 8 total, 7 up, 8 in 2024-09-14T04:17:33.706 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:33 smithi001 ceph-mon[107399]: osdmap e275: 8 total, 7 up, 8 in 2024-09-14T04:17:33.713 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:17:33.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:33.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:33.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:33.980 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:33.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:33.989 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:33.989 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:34.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:34.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:34.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:34.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:34.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:34.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:34.487 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:34.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:34.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:34.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:34.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[107399]: pgmap v995: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 942 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 4737/45258 objects degraded (10.467%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:34.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[107399]: osdmap e276: 8 total, 7 up, 8 in 2024-09-14T04:17:34.669 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4737/45258 objects degraded (10.467%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:34.670 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[112165]: pgmap v995: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 942 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 4737/45258 objects degraded (10.467%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:34.670 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[112165]: osdmap e276: 8 total, 7 up, 8 in 2024-09-14T04:17:34.670 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4737/45258 objects degraded (10.467%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:34.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:34.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:34.764 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:34 smithi012 ceph-mon[101180]: pgmap v995: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 942 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 4737/45258 objects degraded (10.467%); 0 B/s, 71 objects/s recovering 2024-09-14T04:17:34.764 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:34 smithi012 ceph-mon[101180]: osdmap e276: 8 total, 7 up, 8 in 2024-09-14T04:17:34.764 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4737/45258 objects degraded (10.467%), 9 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:34.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:34.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:17:34.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:34.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:34.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:34.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:35.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:35.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:35.201 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:35.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:35.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:35.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.457 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:35.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:35.480 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:17:35.558 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:35.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:35.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:35.666 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:35.666 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:35.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:35.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:35.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:35.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:35.870 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:35.876 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:35.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:35.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:35.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:36.022 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:36.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:36.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:36.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.301 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:36.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:36.437 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:36.509 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[112165]: pgmap v997: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 926 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 4737/44484 objects degraded (10.649%); 0 B/s, 46 objects/s recovering 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[112165]: osdmap e277: 8 total, 7 up, 8 in 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[107399]: pgmap v997: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 926 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 4737/44484 objects degraded (10.649%); 0 B/s, 46 objects/s recovering 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:36.510 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-mon[107399]: osdmap e277: 8 total, 7 up, 8 in 2024-09-14T04:17:36.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:36.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:36.605 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:36.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:36.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:36.756 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:36 smithi012 ceph-mon[101180]: pgmap v997: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 3 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 7 active+undersized+degraded, 69 active+clean; 926 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 4737/44484 objects degraded (10.649%); 0 B/s, 46 objects/s recovering 2024-09-14T04:17:36.756 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:36 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:36.756 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2147609883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:36.756 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:36 smithi012 ceph-mon[101180]: osdmap e277: 8 total, 7 up, 8 in 2024-09-14T04:17:36.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:36.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:36.925 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:36] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.51.0" 2024-09-14T04:17:36.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:36.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:36.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:37.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:37.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:37.137 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:37.142 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:37.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:37.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:37.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:37.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:37.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:37.459 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:37.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:37.588 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:37.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:37.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:37.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:37 smithi001 ceph-mon[107399]: osdmap e278: 8 total, 7 up, 8 in 2024-09-14T04:17:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:37 smithi001 ceph-mon[112165]: osdmap e278: 8 total, 7 up, 8 in 2024-09-14T04:17:37.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:37.741 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:37 smithi012 ceph-mon[101180]: osdmap e278: 8 total, 7 up, 8 in 2024-09-14T04:17:37.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:37.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:37.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:17:37.964 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:37.992 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:37.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:38.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:38.152 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:38.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:38.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:38.337 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:38.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:38.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:38.530 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:38.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:38.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:38 smithi001 ceph-mon[107399]: pgmap v1000: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 691 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 2735/33201 objects degraded (8.238%); 10 MiB/s, 177 objects/s recovering 2024-09-14T04:17:38.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:38 smithi001 ceph-mon[107399]: osdmap e279: 8 total, 7 up, 8 in 2024-09-14T04:17:38.650 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:38 smithi001 ceph-mon[112165]: pgmap v1000: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 691 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 2735/33201 objects degraded (8.238%); 10 MiB/s, 177 objects/s recovering 2024-09-14T04:17:38.650 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:38 smithi001 ceph-mon[112165]: osdmap e279: 8 total, 7 up, 8 in 2024-09-14T04:17:38.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:38.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:38.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:38.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:38 smithi012 ceph-mon[101180]: pgmap v1000: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 691 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 2735/33201 objects degraded (8.238%); 10 MiB/s, 177 objects/s recovering 2024-09-14T04:17:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:38 smithi012 ceph-mon[101180]: osdmap e279: 8 total, 7 up, 8 in 2024-09-14T04:17:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:38.923 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:17:38.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-14T04:17:38.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:38.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:38.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:39.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:39.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:39.218 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:39.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.257 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:39.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:39.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:39.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:39.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:39.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:39.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:39.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:39.694 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:17:39.694 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:17:39.697 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:39.731 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:17:39.731 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:17:39.732 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:39.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:39.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2735/33201 objects degraded (8.238%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:39.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:39 smithi001 ceph-mon[107399]: pgmap v1002: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 675 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2735/32439 objects degraded (8.431%); 8.7 MiB/s, 148 objects/s recovering 2024-09-14T04:17:39.771 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2735/33201 objects degraded (8.238%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:39.771 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:39 smithi001 ceph-mon[112165]: pgmap v1002: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 675 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2735/32439 objects degraded (8.431%); 8.7 MiB/s, 148 objects/s recovering 2024-09-14T04:17:39.771 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:17:39.772 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:17:39.802 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:17:39.802 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:17:39.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:39.823 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:17:39.823 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:17:39.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:17:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2735/33201 objects degraded (8.238%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:39 smithi012 ceph-mon[101180]: pgmap v1002: 99 pgs: 1 active+undersized+remapped, 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 70 active+clean; 675 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2735/32439 objects degraded (8.431%); 8.7 MiB/s, 148 objects/s recovering 2024-09-14T04:17:39.869 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:17:39.869 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:17:39.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:39.908 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:17:39.909 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:17:39.939 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:39.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:40.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:40.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:40.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:40.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:40.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:40.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:40.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:40.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:40.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:40.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:40.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:40.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:40 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3129480590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:40 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3129480590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:40.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:40.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:40 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3129480590' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:40.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:40.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:40.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:40.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:41.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:41.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:41.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:41.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:41.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:41.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:41.330 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:41.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:41.457 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:41.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:41.594 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:41.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:41 smithi012 ceph-mon[101180]: pgmap v1003: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 565 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 2216/27192 objects degraded (8.149%); 8.4 MiB/s, 144 objects/s recovering 2024-09-14T04:17:41.696 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:41.700 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:41 smithi001 ceph-mon[107399]: pgmap v1003: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 565 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 2216/27192 objects degraded (8.149%); 8.4 MiB/s, 144 objects/s recovering 2024-09-14T04:17:41.701 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:41 smithi001 ceph-mon[112165]: pgmap v1003: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 565 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 2216/27192 objects degraded (8.149%); 8.4 MiB/s, 144 objects/s recovering 2024-09-14T04:17:41.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:41.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:41.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:41.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:41.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21879 10.4 10:2c511003:::benchmark_data_smithi001_190881_object21878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:17:42.011 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.05435192699997, 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98974571700000002, 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21879 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:17:42.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953832+0000", 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953830+0000", 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953860+0000", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0428000000000001e-05 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953872+0000", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2091e-05 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953878+0000", 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1179999999999997e-06 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953952+0000", 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3336000000000004e-05 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.953965+0000", 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3721e-05 2024-09-14T04:17:42.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940022+0000", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.986057038 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940143+0000", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121018 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940236+0000", 2024-09-14T04:17:42.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2322000000000001e-05 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943389+0000", 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003153061 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943469+0000", 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0409999999999998e-05 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943481+0000", 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1846e-05 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943537+0000", 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6239000000000001e-05 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943577+0000", 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.002e-05 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21888 10.4 10:2d9d53d0:::benchmark_data_smithi001_190881_object21887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.04911925600004, 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.984828967, 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.019 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21888 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959064+0000", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959063+0000", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959103+0000", 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0454999999999998e-05 2024-09-14T04:17:42.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959118+0000", 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4647999999999999e-05 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959126+0000", 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7489999999999998e-06 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959247+0000", 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121421 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.959274+0000", 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6741e-05 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941501+0000", 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98222701700000004 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941620+0000", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011943999999999999 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.941777+0000", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015683900000000001 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943700+0000", 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019231770000000001 2024-09-14T04:17:42.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943739+0000", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8143999999999999e-05 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943805+0000", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6345000000000003e-05 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943856+0000", 2024-09-14T04:17:42.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0800000000000002e-05 2024-09-14T04:17:42.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943893+0000", 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7509999999999998e-05 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.027 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21892 10.4 10:223580ea:::benchmark_data_smithi001_190881_object21891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.04710172600005, 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.983866295, 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21892 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961082+0000", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961081+0000", 2024-09-14T04:17:42.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961113+0000", 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2369999999999997e-05 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961127+0000", 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3575e-05 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961134+0000", 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.029e-06 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961201+0000", 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7700999999999994e-05 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.961214+0000", 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2417e-05 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942369+0000", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98115569800000002 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942480+0000", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011027900000000001 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942622+0000", 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014186799999999999 2024-09-14T04:17:42.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943796+0000", 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011746479999999999 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943905+0000", 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109042 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944865+0000", 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00096004200000000004 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944914+0000", 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8717999999999997e-05 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944948+0000", 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4029999999999998e-05 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21898 10.4 10:2228a969:::benchmark_data_smithi001_190881_object21897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.04351879800004, 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98440279799999997, 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21898 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964665+0000", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964663+0000", 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964713+0000", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0379e-05 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964726+0000", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.296e-05 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964734+0000", 2024-09-14T04:17:42.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2109999999999998e-06 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964799+0000", 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4608000000000002e-05 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.964811+0000", 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1708999999999999e-05 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943932+0000", 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97912149999999998 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944038+0000", 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010564 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944169+0000", 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013061 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946888+0000", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027196109999999998 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946920+0000", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1519000000000001e-05 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948979+0000", 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020592879999999998 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949026+0000", 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7139999999999999e-05 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949067+0000", 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1396000000000001e-05 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.041 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21908 10.4 10:2c8e17a2:::benchmark_data_smithi001_190881_object21907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.03810430399994, 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97906692100000003, 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21908 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970079+0000", 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970077+0000", 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970145+0000", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7878000000000003e-05 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970162+0000", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6841999999999999e-05 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970169+0000", 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8739999999999997e-06 2024-09-14T04:17:42.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970225+0000", 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5591999999999999e-05 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970236+0000", 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1036000000000001e-05 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944976+0000", 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97474024699999995 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945077+0000", 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101099 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945177+0000", 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9667999999999995e-05 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946948+0000", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00177119 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946976+0000", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7722e-05 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949084+0000", 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021082760000000001 2024-09-14T04:17:42.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949114+0000", 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9964999999999999e-05 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949146+0000", 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2264999999999997e-05 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21912 10.4 10:259478c7:::benchmark_data_smithi001_190881_object21911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.03739993099998, 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97845593200000003, 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21912 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970784+0000", 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970782+0000", 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970811+0000", 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8303999999999999e-05 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970825+0000", 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4528e-05 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970833+0000", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0420000000000003e-06 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970934+0000", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100554 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.970947+0000", 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3183000000000001e-05 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945999+0000", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.975052319 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946112+0000", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112748 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946241+0000", 2024-09-14T04:17:42.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129011 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948925+0000", 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026834599999999999 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948969+0000", 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4746999999999997e-05 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949156+0000", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018685199999999999 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949203+0000", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6959999999999998e-05 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949239+0000", 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6415999999999998e-05 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.055 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21955 10.4 10:23191742:::benchmark_data_smithi001_190881_object21954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 514.01547840199999, 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95687778000000001, 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.056 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21955 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992705+0000", 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992704+0000", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992741+0000", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7347e-05 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992754+0000", 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3004e-05 2024-09-14T04:17:42.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992762+0000", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9799999999999998e-06 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992815+0000", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3074e-05 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.992828+0000", 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2693e-05 2024-09-14T04:17:42.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.946995+0000", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95416745300000005 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947103+0000", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010747 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947219+0000", 2024-09-14T04:17:42.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116579 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949249+0000", 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002029515 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949303+0000", 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4493999999999997e-05 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949394+0000", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0954999999999999e-05 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949450+0000", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5770999999999998e-05 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949583+0000", 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013270900000000001 2024-09-14T04:17:42.062 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22005 10.4 10:249d7abb:::benchmark_data_smithi001_190881_object22004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 513.97148752099997, 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91457057200000003, 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22005 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036696+0000", 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036695+0000", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036741+0000", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6712999999999998e-05 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036753+0000", 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1610000000000001e-05 2024-09-14T04:17:42.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036758+0000", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3249999999999998e-06 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036799+0000", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0995000000000003e-05 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.036807+0000", 2024-09-14T04:17:42.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1119999999999996e-06 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947971+0000", 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.911163905 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948103+0000", 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131345 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:17:42.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.948253+0000", 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014980700000000001 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.950100+0000", 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018479449999999999 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:42.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.950128+0000", 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7174000000000001e-05 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951184+0000", 2024-09-14T04:17:42.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010561119999999999 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951228+0000", 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4464e-05 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951267+0000", 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8315000000000002e-05 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:161525 10.1 10:8b2d5aa2:::benchmark_data_smithi001_190881_object46385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 369.905483629, 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.191356781, 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 161525 2024-09-14T04:17:42.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102698+0000", 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102705+0000", 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0960000000000003e-06 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102720+0000", 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4832e-05 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102727+0000", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3819999999999998e-06 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102789+0000", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1450000000000003e-05 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102903+0000", 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011477899999999999 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103016+0000", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112229 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109259+0000", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062428309999999999 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.111276+0000", 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020170819999999999 2024-09-14T04:17:42.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293948+0000", 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.182672263 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.294010+0000", 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2498999999999999e-05 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.294057+0000", 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6244e-05 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:190078 10.e 10:76b6074a:::benchmark_data_smithi001_190881_object74938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 331.35767776199998, 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28708630899999998, 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.078 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190078 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650504+0000", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650513+0000", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5440000000000004e-06 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650534+0000", 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1009000000000002e-05 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650543+0000", 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1910000000000006e-06 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650591+0000", 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7670000000000003e-05 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650681+0000", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9757000000000001e-05 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650768+0000", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6849999999999997e-05 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.654736+0000", 2024-09-14T04:17:42.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039679140000000003 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.655049+0000", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031341799999999999 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937482+0000", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.282432823 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937542+0000", 2024-09-14T04:17:42.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0265e-05 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937592+0000", 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0016999999999999e-05 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.084 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:190089 10.e 10:7077390a:::benchmark_data_smithi001_190881_object74949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 331.33530653000003, 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26623560299999999, 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190089 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.085 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672876+0000", 2024-09-14T04:17:42.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672883+0000", 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672896+0000", 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3298e-05 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672904+0000", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9279999999999993e-06 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672960+0000", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6360000000000002e-05 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.673104+0000", 2024-09-14T04:17:42.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014360499999999999 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.673206+0000", 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102457 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.676581+0000", 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033750500000000001 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.676693+0000", 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011162000000000001 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939040+0000", 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26234669300000002 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939074+0000", 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3766999999999997e-05 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939113+0000", 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8986000000000002e-05 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:212503 10.e 10:77b7a991:::benchmark_data_smithi001_190881_object97363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 302.04058641199998, 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.138707203, 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.091 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212503 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967595+0000", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967603+0000", 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9559999999999997e-06 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967617+0000", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3786e-05 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967623+0000", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5209999999999999e-06 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967662+0000", 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9100000000000002e-05 2024-09-14T04:17:42.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967775+0000", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112329 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967902+0000", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127465 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.972344+0000", 2024-09-14T04:17:42.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044414340000000002 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.973465+0000", 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001121274 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106206+0000", 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13274113500000001 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106260+0000", 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3634000000000002e-05 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106304+0000", 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4678000000000002e-05 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:212509 10.e 10:796e96a7:::benchmark_data_smithi001_190881_object97369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:17:42.097 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 302.02924492699998, 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.131838491, 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212509 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:17:42.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978937+0000", 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978946+0000", 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0299999999999999e-06 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978961+0000", 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.518e-05 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978968+0000", 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3320000000000003e-06 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979020+0000", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1502000000000003e-05 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979235+0000", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021467200000000001 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979370+0000", 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013521900000000001 2024-09-14T04:17:42.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.985494+0000", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006124394 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.987278+0000", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001783972 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110683+0000", 2024-09-14T04:17:42.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.123405265 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110736+0000", 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2460999999999998e-05 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110777+0000", 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1102000000000001e-05 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111745 11.d 11:b3f626a6:::benchmark_data_smithi001_228075_object111744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 129.578351153, 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0046928020000001, 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111745 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.104 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429831+0000", 2024-09-14T04:17:42.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429860+0000", 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9505e-05 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429872+0000", 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1993e-05 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429880+0000", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5349999999999999e-06 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429939+0000", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9515999999999997e-05 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429947+0000", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0739999999999993e-06 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.432067+0000", 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0021197639999999 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.434525+0000", 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024579860000000001 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111755 11.d 11:b0ede1e3:::benchmark_data_smithi001_228075_object111754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:17:42.108 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 129.57301686900001, 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.006606546, 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111755 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435165+0000", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435200+0000", 2024-09-14T04:17:42.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5256999999999998e-05 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435222+0000", 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2025999999999999e-05 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435232+0000", 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6900000000000004e-06 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435282+0000", 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0390999999999999e-05 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435291+0000", 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1880000000000008e-06 2024-09-14T04:17:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.434878+0000", 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99958683299999995 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.435747+0000", 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00086862599999999995 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.435903+0000", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015575100000000001 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440083+0000", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041805849999999997 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441626+0000", 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015432309999999999 2024-09-14T04:17:42.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441693+0000", 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7076999999999995e-05 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441773+0000", 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9715000000000005e-05 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117525 11.d 11:b15aa863:::benchmark_data_smithi001_228075_object117524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.890887+0000", 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 122.11729667500001, 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.130182039, 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117525 2024-09-14T04:17:42.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890887+0000", 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890887+0000", 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890886+0000", 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890935+0000", 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9357000000000002e-05 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890946+0000", 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0848e-05 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.890953+0000", 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7519999999999999e-06 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.891003+0000", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9851000000000001e-05 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.891117+0000", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011370900000000001 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.891235+0000", 2024-09-14T04:17:42.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011853099999999999 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902086+0000", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010850999 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.017141+0000", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.115054908 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.020972+0000", 2024-09-14T04:17:42.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038311999999999999 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.021069+0000", 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6712999999999994e-05 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117532 11.d 11:ba26fc87:::benchmark_data_smithi001_228075_object117531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 122.110079688, 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13377942700000001, 2024-09-14T04:17:42.123 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117532 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898103+0000", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898166+0000", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3083000000000006e-05 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898176+0000", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0638999999999999e-05 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898184+0000", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0450000000000003e-06 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902114+0000", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039306489999999996 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902263+0000", 2024-09-14T04:17:42.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014865000000000001 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902410+0000", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147465 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.021140+0000", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.118729843 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031763+0000", 2024-09-14T04:17:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010622509 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031809+0000", 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6316000000000002e-05 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031883+0000", 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4270999999999994e-05 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.128 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117538 11.d 11:b56e9153:::benchmark_data_smithi001_228075_object117537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 122.105430039, 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13954377000000001, 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117538 2024-09-14T04:17:42.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902752+0000", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902792+0000", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0565999999999998e-05 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.013440+0000", 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11064774600000001 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.013457+0000", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6597999999999999e-05 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.021164+0000", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0077068120000000004 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.022584+0000", 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014198679999999999 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.022732+0000", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014866 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031891+0000", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091587700000000001 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.041218+0000", 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093267009999999997 2024-09-14T04:17:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.041275+0000", 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7840000000000002e-05 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.042297+0000", 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010217340000000001 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.134 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:171808 11.e 11:712bfe5c:::benchmark_data_smithi001_228075_object53493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:16:35.555132+0000", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 66.453051305000002, 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098738708999999994, 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 171808 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555132+0000", 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555132+0000", 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555131+0000", 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555135+0000", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4730000000000002e-06 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555151+0000", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5098e-05 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.555158+0000", 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0840000000000003e-06 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.558570+0000", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034126149999999999 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.558895+0000", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032488999999999999 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.559424+0000", 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00052908999999999996 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.564435+0000", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050109610000000004 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.564479+0000", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4165999999999997e-05 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.653761+0000", 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089281481999999995 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.653830+0000", 2024-09-14T04:17:42.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8955000000000006e-05 2024-09-14T04:17:42.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:35.653871+0000", 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1069999999999998e-05 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.141 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:212104 11.e 11:7eb88606:::benchmark_data_smithi001_228075_object93789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:21.630695+0000", 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 20.377488653, 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064680199999999993, 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:17:42.142 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212104 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630695+0000", 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630695+0000", 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:42.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630693+0000", 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630701+0000", 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6389999999999992e-06 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630716+0000", 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5840000000000001e-05 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630725+0000", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0459999999999994e-06 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630778+0000", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2481000000000001e-05 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.630895+0000", 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116857 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.631005+0000", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110111 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.635481+0000", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044756550000000003 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.635517+0000", 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6399000000000002e-05 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.695289+0000", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059772118999999999 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.695336+0000", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7259000000000003e-05 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.695375+0000", 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8729000000000001e-05 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.147 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.148 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:42.148 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:42.148 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:17:42.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:42.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:42.205 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:17:42.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:42.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:42.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:42.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:42.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:42.610 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:42.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:42.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:42.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:42.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:42.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:42.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:42.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:43.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:17:43.034 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:43.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:43.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:43.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.343 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:43.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:43.449 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:43.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:43.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:43.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:43.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:43.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:43.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:43.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:44.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:44.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:44.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:44.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:17:44.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:44.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:44.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:44.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:44.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:44.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:44.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:44 smithi012 ceph-mon[101180]: pgmap v1004: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 438 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 1749/21081 objects degraded (8.297%); 6.5 MiB/s, 110 objects/s recovering 2024-09-14T04:17:44.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 1749/21081 objects degraded (8.297%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:44.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:44 smithi001 ceph-mon[112165]: pgmap v1004: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 438 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 1749/21081 objects degraded (8.297%); 6.5 MiB/s, 110 objects/s recovering 2024-09-14T04:17:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 1749/21081 objects degraded (8.297%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:44 smithi001 ceph-mon[107399]: pgmap v1004: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 438 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 1749/21081 objects degraded (8.297%); 6.5 MiB/s, 110 objects/s recovering 2024-09-14T04:17:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 1749/21081 objects degraded (8.297%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:44.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:44.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:44.866 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:17:44.867 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:17:44.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:44.909 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:44.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:45.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:45.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:45.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:45.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:45.192 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:45.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:45.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:45.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:45.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:45.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:45.609 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:45.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:45.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:45.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:45.900 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:45.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:17:45.903 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:17:45.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-14T04:17:46.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:46.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:46.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:46.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:46.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:46.347 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:46.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:46.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:46.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:46.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:46 smithi012 ceph-mon[101180]: pgmap v1005: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 419 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1749/20169 objects degraded (8.672%); 0 B/s, 6 objects/s recovering 2024-09-14T04:17:46.599 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:46.638 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:46.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:46 smithi001 ceph-mon[107399]: pgmap v1005: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 419 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1749/20169 objects degraded (8.672%); 0 B/s, 6 objects/s recovering 2024-09-14T04:17:46.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:46 smithi001 ceph-mon[112165]: pgmap v1005: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 419 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1749/20169 objects degraded (8.672%); 0 B/s, 6 objects/s recovering 2024-09-14T04:17:46.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:46.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:46.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:46.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:46.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:46.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:46] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.51.0" 2024-09-14T04:17:47.022 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:47.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:47.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:47.166 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:47.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:47.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:47.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:47.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:47.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:47.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:47.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:47.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:47.718 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:47.756 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:47.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:47.904 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:47.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:48.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35451.0:236181 11.3 e279/275 11:c7aecef2:::benchmark_data_smithi001_228075_object117866:head v 279'14626, mlcod=279'14622)", 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:48.010714+0000", 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0067909010000000002, 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068213969999999999, 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:48.019 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010714+0000", 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010714+0000", 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010713+0000", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010718+0000", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4689999999999996e-06 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010736+0000", 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7600000000000001e-05 2024-09-14T04:17:48.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010744+0000", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8380000000000002e-06 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010900+0000", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015610299999999999 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010927+0000", 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7092000000000001e-05 2024-09-14T04:17:48.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35451.0:236183 11.3 e279/275 11:ca17956d:::benchmark_data_smithi001_228075_object117868:head v 279'14627, mlcod=279'14622)", 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:48.010759+0000", 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0067457539999999996, 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068404870000000001, 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010759+0000", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010759+0000", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010759+0000", 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010763+0000", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6589999999999998e-06 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010773+0000", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0026e-05 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.010779+0000", 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8980000000000001e-06 2024-09-14T04:17:48.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.014987+0000", 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042075949999999997 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.015005+0000", 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8355e-05 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.026 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35451.0:236200 11.3 e279/275)", 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:48.012293+0000", 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0052120220000000002, 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053606069999999999, 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012293+0000", 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012293+0000", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012292+0000", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012299+0000", 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0990000000000001e-06 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012315+0000", 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6195e-05 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.012322+0000", 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3550000000000001e-06 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:17:48.029 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2024-09-14T04:17:48.030 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:17:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.036 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:48.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:48.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:48.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:48.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:48.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:48.379 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:17:48.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:48.451 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:48.513 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 118312 objects 2024-09-14T04:17:48.513 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :127.712 2024-09-14T04:17:48.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:48.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:48 smithi012 ceph-mon[101180]: pgmap v1006: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 132 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 544/6408 objects degraded (8.489%) 2024-09-14T04:17:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:48 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2182428534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:48 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.676 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:48.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[107399]: pgmap v1006: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 132 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 544/6408 objects degraded (8.489%) 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2182428534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[112165]: pgmap v1006: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 132 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 544/6408 objects degraded (8.489%) 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2182428534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:48 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:17:48.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:48.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:48.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:48.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:48.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:49.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:49.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:49.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:49.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:49.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:49.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:49.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:49.447 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:49.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[112165]: osdmap e280: 8 total, 7 up, 8 in 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 544/6408 objects degraded (8.489%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[107399]: osdmap e280: 8 total, 7 up, 8 in 2024-09-14T04:17:49.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 544/6408 objects degraded (8.489%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:49.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:17:49.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:49.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:49.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:49.737 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:49.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:49.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:49 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:17:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:49 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:17:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:49 smithi012 ceph-mon[101180]: osdmap e280: 8 total, 7 up, 8 in 2024-09-14T04:17:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 544/6408 objects degraded (8.489%), 7 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:49.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:49.946 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:49.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:50.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:50.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:50.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:50.194 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:50.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:50.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:50.356 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-09-14T04:17:50.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-09-14T04:17:50.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:50.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:50.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:50.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:50.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:50.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:50.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:50 smithi001 ceph-mon[107399]: pgmap v1008: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 116 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 544/5619 objects degraded (9.681%) 2024-09-14T04:17:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:50 smithi001 ceph-mon[112165]: pgmap v1008: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 116 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 544/5619 objects degraded (9.681%) 2024-09-14T04:17:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:50.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:50.803 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:50 smithi012 ceph-mon[101180]: pgmap v1008: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 7 active+undersized+degraded, 71 active+clean; 116 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 544/5619 objects degraded (9.681%) 2024-09-14T04:17:50.803 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:50.812 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:50.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:50.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:50.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:51.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:51.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:51.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:51.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:51.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.310 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:51.365 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:17:51.365 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:17:51.365 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:17:51.365 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:17:51.365 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:17:51.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:51.433 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:51.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:51.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:51.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:51.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:51.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:17:51.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:51.902 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:17:51.903 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:17:51.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:51.911 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:51.914 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:17:51.914 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:17:51.915 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:17:51.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:51.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:51.943 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:17:51.943 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:17:51.960 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:17:51.960 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:17:51.976 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:17:51.976 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:17:51.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:17:51.993 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:17:51.993 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:17:52.009 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:17:52.009 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:17:52.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:52.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:52.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:52.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:52.195 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:52.285 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:52.308 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:52.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:52.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:52 smithi001 ceph-mon[107399]: pgmap v1009: 99 pgs: 3 active+clean+remapped, 19 active+undersized, 6 active+undersized+degraded, 71 active+clean; 48 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 224/2361 objects degraded (9.488%) 2024-09-14T04:17:52.429 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3699746572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:52.430 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:52 smithi001 ceph-mon[112165]: pgmap v1009: 99 pgs: 3 active+clean+remapped, 19 active+undersized, 6 active+undersized+degraded, 71 active+clean; 48 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 224/2361 objects degraded (9.488%) 2024-09-14T04:17:52.430 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3699746572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:52.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:52.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:52.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:52.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:52 smithi012 ceph-mon[101180]: pgmap v1009: 99 pgs: 3 active+clean+remapped, 19 active+undersized, 6 active+undersized+degraded, 71 active+clean; 48 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 224/2361 objects degraded (9.488%) 2024-09-14T04:17:52.589 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3699746572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:17:52.657 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:17:52.658 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:17:52.658 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:17:52.658 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:17:52.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:52.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:52.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:52.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:52.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:52.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:52.914 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:52.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:52.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:52.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:53.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:53.062 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:53.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:53.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.188 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:53.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:53.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:53.314 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:53.416 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:53.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:53.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:53.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:53.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:53.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:53.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:53.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:53.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:53.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:53.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:54.002 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:54.002 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:54.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:17:54.065 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:17:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:54.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:54.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:54.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:54.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:54.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:54.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:54.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:54.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:54.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:54.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:54 smithi001 ceph-mon[107399]: pgmap v1010: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 5 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:54 smithi001 ceph-mon[112165]: pgmap v1010: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 5 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:54.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:54.827 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:54 smithi012 ceph-mon[101180]: pgmap v1010: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 5 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:54.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:54.952 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:54.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:54.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:55.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:55.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:55.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:17:55.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:55.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:55.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:55.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:55.370 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:55.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:55.450 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:55.474 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:55.493 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:17:55.493 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:17:55.494 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-14T04:17:55.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:55.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:55.676 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:55.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:55.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:55.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:55.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:55.952 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:55.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:56.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:56.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:17:56.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:56.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:56.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:56.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:56.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:56.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:56.396 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_6' removed 2024-09-14T04:17:56.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:56.450 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:56.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:56 smithi012 ceph-mon[101180]: pgmap v1011: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:56.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3286737005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.519 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:56 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:56.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:56.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[112165]: pgmap v1011: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3286737005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[107399]: pgmap v1011: 99 pgs: 3 active+clean+remapped, 20 active+undersized, 5 active+undersized+degraded, 71 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 4/78 objects degraded (5.128%) 2024-09-14T04:17:56.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3286737005' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:17:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:56.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:56.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:56.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:56.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:56.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:17:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:17:56] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-09-14T04:17:57.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:57.059 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:57.059 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:57.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:57.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:57.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:57.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:57.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:57.428 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:57.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:57.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:57.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:57.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:57.660 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:57 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:17:57.661 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:57 smithi012 ceph-mon[101180]: osdmap e281: 8 total, 7 up, 8 in 2024-09-14T04:17:57.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:17:57.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:17:57.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:57.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:17:57.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:57 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:17:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:57 smithi001 ceph-mon[112165]: osdmap e281: 8 total, 7 up, 8 in 2024-09-14T04:17:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:57 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:17:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:57 smithi001 ceph-mon[107399]: osdmap e281: 8 total, 7 up, 8 in 2024-09-14T04:17:57.785 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:17:57.786 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:17:57.787 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2024-09-14T04:17:57.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_7 16 2024-09-14T04:17:57.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:57.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:57.906 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:17:57.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:57.976 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:57.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:58.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:58.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:58.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:58.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:58.231 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:58.233 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:17:58.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:58.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:58.405 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:17:58.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:58.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:58.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:58.598 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:58.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:58 smithi001 ceph-mon[107399]: pgmap v1013: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:17:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:58 smithi001 ceph-mon[112165]: pgmap v1013: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:17:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:58.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:17:58.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:58.834 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:58 smithi012 ceph-mon[101180]: pgmap v1013: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:17:58.834 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:17:58.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:58.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:58.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:58.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:59.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:17:59.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:17:59.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:17:59.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:17:59.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:59.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:59.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:59.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[107399]: osdmap e282: 8 total, 7 up, 8 in 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[112165]: osdmap e282: 8 total, 7 up, 8 in 2024-09-14T04:17:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:17:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:59.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:17:59.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:17:59.656 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:59 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:17:59.656 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3219812421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:17:59.657 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:59 smithi012 ceph-mon[101180]: osdmap e282: 8 total, 7 up, 8 in 2024-09-14T04:17:59.657 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:17:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:17:59.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:17:59.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:17:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:17:59.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:17:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:59.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:17:59.858 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:17:59.858 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:17:59.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:17:59.967 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:17:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:17:59.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:00.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.092 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:00.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:00.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:00.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:00.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:00.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.344 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:00.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:00.434 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:00 smithi012 ceph-mon[101180]: pgmap v1015: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:00.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:00.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:18:00.467 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:00.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:00 smithi001 ceph-mon[107399]: pgmap v1015: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:00 smithi001 ceph-mon[112165]: pgmap v1015: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:00.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:00.591 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:00.663 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:18:00.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:00.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:00.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:00.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:00.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:00.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:00.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:01.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:01.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:01.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:18:01.192 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:01.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:01.301 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:18:01.301 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:18:01.313 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:18:01.313 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:18:01.313 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:18:01.332 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:18:01.333 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:18:01.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:01.349 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:18:01.349 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:18:01.358 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:01.366 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:18:01.366 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:18:01.383 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:18:01.383 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:18:01.399 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:18:01.399 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:18:01.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:01.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1961779950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:01.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1961779950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:01.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:01.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:01.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:01.709 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:01.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:01.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:01.767 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1961779950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:01.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:01.857 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:01.959 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:01.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:01.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:02.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:02.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:02.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:02.230 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:02.275 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:18:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:18:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:18:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:18:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21703 10.7 10:e076e219:::benchmark_data_smithi001_190881_object21702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 535.32337379700004, 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99404368700000001, 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21703 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948728+0000", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948727+0000", 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948757+0000", 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0284e-05 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948770+0000", 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3258000000000001e-05 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948777+0000", 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9789999999999996e-06 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948870+0000", 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2341000000000002e-05 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:06.948884+0000", 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4677e-05 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939652+0000", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99076829799999999 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939754+0000", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101926 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.939848+0000", 2024-09-14T04:18:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3722999999999997e-05 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942664+0000", 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028159140000000001 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942684+0000", 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9796000000000002e-05 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942706+0000", 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2467e-05 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942734+0000", 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8016000000000001e-05 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.942771+0000", 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7021999999999999e-05 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21874 10.a 10:50ff48f0:::benchmark_data_smithi001_190881_object21873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 534.321527264, 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99320812599999997, 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21874 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950574+0000", 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950573+0000", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950634+0000", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1093999999999995e-05 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950645+0000", 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0852999999999999e-05 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950652+0000", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8260000000000004e-06 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950713+0000", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0912999999999998e-05 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.950723+0000", 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0582e-05 2024-09-14T04:18:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.939710+0000", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98898640699999996 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.939878+0000", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168376 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.940037+0000", 2024-09-14T04:18:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015902599999999999 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943506+0000", 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034692540000000002 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943622+0000", 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011597800000000001 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943688+0000", 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5625999999999998e-05 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943745+0000", 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6786999999999997e-05 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943782+0000", 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7608000000000003e-05 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21883 10.a 10:514bc832:::benchmark_data_smithi001_190881_object21882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 534.316108859, 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98944681199999995, 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21883 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955993+0000", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.955992+0000", 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956024+0000", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2286000000000002e-05 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956039+0000", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4662e-05 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956047+0000", 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7879999999999995e-06 2024-09-14T04:18:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956099+0000", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1857000000000002e-05 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.956109+0000", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6940000000000007e-06 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942527+0000", 2024-09-14T04:18:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98641796100000001 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942650+0000", 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000123328 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.942843+0000", 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019251899999999999 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944044+0000", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001201559 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945324+0000", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012796439999999999 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945359+0000", 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4938999999999999e-05 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945401+0000", 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1993000000000003e-05 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.945439+0000", 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8507999999999998e-05 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:21887 10.a 10:5d6c9b5d:::benchmark_data_smithi001_190881_object21886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 534.31334789899995, 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99076259499999997, 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 21887 2024-09-14T04:18:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958754+0000", 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958749+0000", 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958794+0000", 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4885000000000001e-05 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958805+0000", 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.093e-05 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958812+0000", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.255e-06 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958865+0000", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2920999999999997e-05 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:07.958874+0000", 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2730000000000008e-06 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944101+0000", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98522702500000003 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944185+0000", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3516000000000005e-05 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944317+0000", 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132807 2024-09-14T04:18:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947650+0000", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033321980000000002 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.947717+0000", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7151999999999997e-05 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949433+0000", 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001715928 2024-09-14T04:18:02.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949477+0000", 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4487999999999998e-05 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.949516+0000", 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8992000000000002e-05 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.304 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22006 10.d 10:b8ac2021:::benchmark_data_smithi001_190881_object22005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 533.32829213599996, 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0147531949999999, 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22006 2024-09-14T04:18:02.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943809+0000", 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943808+0000", 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943951+0000", 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014249799999999999 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943965+0000", 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4518999999999999e-05 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.943975+0000", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9129999999999998e-06 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944044+0000", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8853e-05 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.944053+0000", 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1870000000000003e-06 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.954838+0000", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0107845520000001 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.954995+0000", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015683799999999999 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.955105+0000", 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110714 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958327+0000", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003222043 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958339+0000", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1308000000000001e-05 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958465+0000", 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012594599999999999 2024-09-14T04:18:02.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958518+0000", 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3028000000000001e-05 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958563+0000", 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4975000000000002e-05 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:22019 10.d 10:bd024054:::benchmark_data_smithi001_190881_object22018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 533.32096292799997, 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.00765544, 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 22019 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951139+0000", 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951137+0000", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951170+0000", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2492e-05 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951530+0000", 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036076700000000001 2024-09-14T04:18:02.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951539+0000", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6500000000000002e-06 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951655+0000", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115574 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:08.951698+0000", 2024-09-14T04:18:02.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3602e-05 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956415+0000", 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.004716731 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956562+0000", 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014654499999999999 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.956725+0000", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000163361 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958620+0000", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001894939 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958629+0000", 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8559999999999996e-06 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958693+0000", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4189000000000005e-05 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958749+0000", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5680999999999998e-05 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:09.958794+0000", 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5371000000000003e-05 2024-09-14T04:18:02.318 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85152 10.0 10:04bd257d:::benchmark_data_smithi001_190881_object85151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.05701579200002, 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2469428599999999, 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.319 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85152 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215086+0000", 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215084+0000", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215117+0000", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3624000000000002e-05 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215134+0000", 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6758000000000001e-05 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215143+0000", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5080000000000005e-06 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215208+0000", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4907999999999996e-05 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215356+0000", 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148114 2024-09-14T04:18:02.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.215492+0000", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013663999999999999 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.224934+0000", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094411930000000005 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.224947+0000", 2024-09-14T04:18:02.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3093e-05 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215613+0000", 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99066625600000002 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215623+0000", 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0377000000000001e-05 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.446326+0000", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.230702956 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.446898+0000", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00057135599999999997 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461915+0000", 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015017536 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462029+0000", 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011345000000000001 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85170 10.0 10:0452c479:::benchmark_data_smithi001_190881_object85169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.02871765700002, 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2236046439999999, 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.326 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85170 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243384+0000", 2024-09-14T04:18:02.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243379+0000", 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243425+0000", 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5998999999999998e-05 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243438+0000", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3225e-05 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243446+0000", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7610000000000007e-06 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243527+0000", 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1230999999999995e-05 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.243540+0000", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2397e-05 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215644+0000", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97210399800000002 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215650+0000", 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3430000000000001e-06 2024-09-14T04:18:02.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449443+0000", 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2337928069999999 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449597+0000", 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015371499999999999 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:18:02.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449766+0000", 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016950299999999999 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462179+0000", 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012412929 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462187+0000", 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5109999999999997e-06 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462254+0000", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7372999999999993e-05 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462358+0000", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103946 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.466988+0000", 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046306300000000002 2024-09-14T04:18:02.333 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85185 10.7 10:e9b0eeb6:::benchmark_data_smithi001_190881_object85184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.01763633000002, 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2214390960000001, 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.334 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85185 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254465+0000", 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254507+0000", 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2392999999999999e-05 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254517+0000", 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0251999999999999e-05 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254524+0000", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4910000000000002e-06 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254566+0000", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2171e-05 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.254574+0000", 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1370000000000002e-06 2024-09-14T04:18:02.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214574+0000", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95999957999999996 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214587+0000", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3329999999999999e-05 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449262+0000", 2024-09-14T04:18:02.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.234674703 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449425+0000", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016336700000000001 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.449533+0000", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108441 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458502+0000", 2024-09-14T04:18:02.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089684389999999999 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458552+0000", 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0621999999999997e-05 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471592+0000", 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013039258 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471645+0000", 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3643000000000001e-05 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475904+0000", 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042588529999999999 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85187 10.7 10:e711136f:::benchmark_data_smithi001_190881_object85186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.341 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.01491767800002, 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2358304539999998, 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85187 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257184+0000", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257182+0000", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257217+0000", 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4579999999999998e-05 2024-09-14T04:18:02.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257237+0000", 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9930999999999999e-05 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257245+0000", 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3510000000000002e-06 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257373+0000", 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012805 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.257384+0000", 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0361e-05 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214610+0000", 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95722658400000005 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214618+0000", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5800000000000003e-06 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455570+0000", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2409521269999999 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455741+0000", 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017140699999999999 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.455928+0000", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018640099999999999 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464920+0000", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089924170000000008 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464981+0000", 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0770999999999998e-05 2024-09-14T04:18:02.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.489152+0000", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024170921000000001 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.489282+0000", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013016699999999999 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493014+0000", 2024-09-14T04:18:02.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003732366 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85193 10.7 10:e361fc0b:::benchmark_data_smithi001_190881_object85192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.01233904200001, 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2371485259999999, 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.349 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85193 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259762+0000", 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259761+0000", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259817+0000", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5583e-05 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259827+0000", 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8849999999999994e-06 2024-09-14T04:18:02.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259833+0000", 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8499999999999999e-06 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259874+0000", 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1316000000000002e-05 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.259880+0000", 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.032e-06 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214639+0000", 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95475853099999997 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214647+0000", 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9829999999999996e-06 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.353 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458633+0000", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2439862399999999 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458791+0000", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157637 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.458961+0000", 2024-09-14T04:18:02.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000170056 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464971+0000", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006010329 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.464986+0000", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5319999999999999e-05 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493078+0000", 2024-09-14T04:18:02.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02809124 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.493147+0000", 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9715000000000006e-05 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496911+0000", 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037637360000000002 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.356 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85196 10.7 10:e2c2e82c:::benchmark_data_smithi001_190881_object85195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.00955617800003, 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2380154719999998, 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85196 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.357 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262545+0000", 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262544+0000", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262604+0000", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9871000000000003e-05 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262614+0000", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0040999999999999e-05 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262621+0000", 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9129999999999997e-06 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262656+0000", 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5082000000000001e-05 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.262662+0000", 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1210000000000003e-06 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214668+0000", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95200554699999995 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214684+0000", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6385e-05 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461526+0000", 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2468417430000001 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.461711+0000", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018453799999999999 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.462421+0000", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00071004800000000002 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.471513+0000", 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090922599999999996 2024-09-14T04:18:02.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475962+0000", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044492680000000001 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496962+0000", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020999752 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.496999+0000", 2024-09-14T04:18:02.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6591000000000003e-05 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500561+0000", 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003562164 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85205 10.7 10:ecddaef8:::benchmark_data_smithi001_190881_object85204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:18:02.364 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.00132326599999, 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2339501529999999, 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85205 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270778+0000", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270814+0000", 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6078999999999998e-05 2024-09-14T04:18:02.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270824+0000", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0349999999999999e-05 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270830+0000", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.331e-06 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270879+0000", 2024-09-14T04:18:02.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9149e-05 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.270888+0000", 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4330000000000004e-06 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214707+0000", 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94381939299999995 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.214715+0000", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8250000000000005e-06 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465015+0000", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2502998060000001 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465190+0000", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017487000000000001 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.465364+0000", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017457099999999999 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.475998+0000", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01063412 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.476064+0000", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5627e-05 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500619+0000", 2024-09-14T04:18:02.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024554630000000001 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.500671+0000", 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2207999999999999e-05 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.504728+0000", 2024-09-14T04:18:02.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040574410000000002 2024-09-14T04:18:02.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85212 10.0 10:05efc25a:::benchmark_data_smithi001_190881_object85211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 492.99366396300002, 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.207828165, 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.373 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85212 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278438+0000", 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278436+0000", 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278496+0000", 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9991999999999998e-05 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278509+0000", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2668e-05 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278517+0000", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1019999999999997e-06 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278572+0000", 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5083000000000001e-05 2024-09-14T04:18:02.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.278583+0000", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0424e-05 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215667+0000", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93708416800000005 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215684+0000", 2024-09-14T04:18:02.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6939000000000002e-05 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467030+0000", 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2513462829999999 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467188+0000", 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157966 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:18:02.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.467374+0000", 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018620299999999999 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.477554+0000", 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010179911999999999 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482157+0000", 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046033660000000002 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482229+0000", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1594000000000006e-05 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482410+0000", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018091900000000001 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.486266+0000", 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038557079999999998 2024-09-14T04:18:02.380 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:85226 10.0 10:083bf9b5:::benchmark_data_smithi001_190881_object85225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 492.97516239499998, 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2170019399999998, 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.381 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85226 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296939+0000", 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296938+0000", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296984+0000", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6419999999999999e-05 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.296999+0000", 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4878999999999999e-05 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297007+0000", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6459999999999993e-06 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297071+0000", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4008000000000001e-05 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:49.297083+0000", 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2167999999999999e-05 2024-09-14T04:18:02.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215705+0000", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91862224100000001 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:50.215712+0000", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6080000000000001e-06 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470415+0000", 2024-09-14T04:18:02.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2547035580000001 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470574+0000", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015836499999999999 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.470739+0000", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000165663 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.482192+0000", 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011453029999999999 2024-09-14T04:18:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.490246+0000", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.008053859 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.511198+0000", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020951908000000002 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.511262+0000", 2024-09-14T04:18:02.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4046000000000003e-05 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:09:51.513941+0000", 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026789460000000002 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:216563 10.7 10:e4c6ac6a:::benchmark_data_smithi001_190881_object101423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:45.301258+0000", 2024-09-14T04:18:02.388 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 316.97084355599998, 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.016764139, 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216563 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301258+0000", 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301258+0000", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301257+0000", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301262+0000", 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4240000000000001e-06 2024-09-14T04:18:02.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301274+0000", 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1613999999999999e-05 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301281+0000", 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7530000000000001e-06 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301361+0000", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9561000000000006e-05 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.301372+0000", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0806e-05 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.307904+0000", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.006532647 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.307994+0000", 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0013999999999996e-05 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.308078+0000", 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3660999999999997e-05 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.317828+0000", 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0097500539999999993 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.317918+0000", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9400000000000005e-05 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.317949+0000", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1281000000000001e-05 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.317984+0000", 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5271000000000002e-05 2024-09-14T04:18:02.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318022+0000", 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7945999999999999e-05 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:216583 10.7 10:edd715cb:::benchmark_data_smithi001_190881_object101443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:45.326145+0000", 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 316.94595656299998, 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99199923800000001, 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.395 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216583 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326145+0000", 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326145+0000", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326143+0000", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326152+0000", 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4210000000000001e-06 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326167+0000", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4109e-05 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326177+0000", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.065e-05 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326241+0000", 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3380999999999995e-05 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:45.326253+0000", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2204999999999999e-05 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.311228+0000", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98497498100000003 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.311355+0000", 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012697300000000001 2024-09-14T04:18:02.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.311492+0000", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013769300000000001 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318034+0000", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0065412179999999997 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318067+0000", 2024-09-14T04:18:02.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3275000000000002e-05 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318097+0000", 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9598e-05 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318121+0000", 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4901000000000001e-05 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:46.318144+0000", 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2670000000000001e-05 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:232750 11.8 11:1abf01bc:::benchmark_data_smithi001_228075_object114435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:44.512184+0000", 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 17.759917466000001, 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082593604000000001, 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 232750 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512184+0000", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512184+0000", 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512183+0000", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512188+0000", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2979999999999999e-06 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512198+0000", 2024-09-14T04:18:02.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0112000000000001e-05 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512205+0000", 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6220000000000003e-06 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512313+0000", 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108316 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512428+0000", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115157 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.512493+0000", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5180999999999998e-05 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.516447+0000", 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003954089 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.517069+0000", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062207199999999999 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.594683+0000", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077613737000000002 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.594749+0000", 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5788000000000001e-05 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.594778+0000", 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8651000000000001e-05 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:232765 11.8 11:14e797a3:::benchmark_data_smithi001_228075_object114450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:44.530565+0000", 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 17.741536851999999, 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.087165962999999999, 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.408 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 232765 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.530565+0000", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.530565+0000", 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.530564+0000", 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.530568+0000", 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6099999999999999e-06 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.588667+0000", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058098505000000002 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.588714+0000", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7307e-05 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.594950+0000", 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062361059999999999 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.595101+0000", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150168 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.595197+0000", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6778000000000001e-05 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.606663+0000", 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011466057 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.606816+0000", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015310500000000001 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.617640+0000", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010823263 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.617694+0000", 2024-09-14T04:18:02.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3918000000000001e-05 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:44.617731+0000", 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6924000000000001e-05 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:236456 11.1 11:8d4a2d42:::benchmark_data_smithi001_228075_object118141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:48.287379+0000", 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 13.984722047, 2024-09-14T04:18:02.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.091322367000000002, 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 236456 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287379+0000", 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287379+0000", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287378+0000", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287390+0000", 2024-09-14T04:18:02.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1919999999999999e-05 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287402+0000", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1926999999999999e-05 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.287410+0000", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6399999999999997e-06 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.296353+0000", 2024-09-14T04:18:02.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089437939999999997 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.296504+0000", 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151139 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.296624+0000", 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119181 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.303464+0000", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068398909999999999 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.303618+0000", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015408500000000001 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.378611+0000", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.074993776999999998 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:02.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.378658+0000", 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6579000000000002e-05 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:48.378702+0000", 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3832999999999999e-05 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:02.420 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:18:02.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:02.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:02.501 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:18:02.502 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:18:02.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:02.566 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:02.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:02 smithi012 ceph-mon[101180]: pgmap v1016: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:02.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:02.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:02.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:02 smithi001 ceph-mon[107399]: pgmap v1016: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:02 smithi001 ceph-mon[112165]: pgmap v1016: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:02.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:02.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:02.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:02.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:02.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:03.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:03.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:03.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:03.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:03.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:03.359 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:18:03.360 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:18:03.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-14T04:18:03.383 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:18:03.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:03.448 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:03.478 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:03.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:03.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:03.587 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:18:03.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:03.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:03.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:03.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:03.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:03.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:03.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:03.972 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:18:03.973 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:18:04.043 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:04.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:04.057 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:04.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:04.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:04.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:04.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:04.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:04.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:04.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:04.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:04.563 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:04 smithi001 ceph-mon[107399]: pgmap v1017: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:04.563 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:04 smithi001 ceph-mon[112165]: pgmap v1017: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:04.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:04.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:04.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:04.735 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:04 smithi012 ceph-mon[101180]: pgmap v1017: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:04.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:04.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:04.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:04.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:04.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:05.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:05.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:05.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:05.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:05.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:05.310 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:18:05.310 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:18:05.310 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2024-09-14T04:18:05.310 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:18:05.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:05.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:05.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:05.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:05.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:05.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:05.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:05.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:05.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:05.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:05.956 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:05.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:05.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:18:05.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:18:06.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:06.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:06.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:06.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:06.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[107399]: pgmap v1018: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:05 smithi001 ceph-mon[112165]: pgmap v1018: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:06.225 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:18:06.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:06.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:05 smithi012 ceph-mon[101180]: pgmap v1018: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:06.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:06.456 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:06.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:06.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:06.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:06.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:06.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:06.832 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:06.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:06.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:06.958 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:06.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:06] "GET /metrics HTTP/1.1" 200 34211 "" "Prometheus/2.51.0" 2024-09-14T04:18:07.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:07.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:07.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:07.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:07.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:07.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:07.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:07.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:07.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:07.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:07.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:07.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:07.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:07.881 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:07.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:07.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:07.929 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:18:07.929 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:18:07.929 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:18:07.929 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:18:07.929 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:18:07.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:08.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:08.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:08.130 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:08.136 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:08.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:08.254 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:08.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:08.371 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_7' created 2024-09-14T04:18:08.377 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:18:08.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:08.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:08.509 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:08.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:08 smithi012 ceph-mon[101180]: pgmap v1019: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:08.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-14T04:18:08.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3773171172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.551 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:08 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:18:08.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:08.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[107399]: pgmap v1019: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-14T04:18:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3773171172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[112165]: pgmap v1019: 83 pgs: 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-14T04:18:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3773171172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:08 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:18:08.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:08.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:08.818 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:18:08.862 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:08.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:08.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:08.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:08.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:09.007 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:09.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:09.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:09.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:09.256 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:09.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:09.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:09.492 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:09.515 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:09 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-14T04:18:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:18:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:09 smithi012 ceph-mon[101180]: osdmap e283: 8 total, 7 up, 8 in 2024-09-14T04:18:09.617 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:09.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:09.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:09.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-14T04:18:09.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:18:09.648 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[107399]: osdmap e283: 8 total, 7 up, 8 in 2024-09-14T04:18:09.648 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:09.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2108248591' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-14T04:18:09.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:18:09.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:09 smithi001 ceph-mon[112165]: osdmap e283: 8 total, 7 up, 8 in 2024-09-14T04:18:09.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2024-09-14T04:18:09.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:09.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:09.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:09.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:09.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:18:09.913 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:18:09.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:10.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:10.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:10.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:10.237 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:10.261 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:10.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:10.386 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45035.0:161525 10.1 e225/184 10:8b2d5aa2:::benchmark_data_smithi001_190881_object46385:head v 225'10154, mlcod=225'10152)", 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 398.34529852200001, 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.190663473, 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.470 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103144+0000", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103143+0000", 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103151+0000", 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7330000000000003e-06 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103162+0000", 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0943e-05 2024-09-14T04:18:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103169+0000", 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9859999999999997e-06 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263421+0000", 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16025208599999999 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.263445+0000", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3892000000000001e-05 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293726+0000", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030281499 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293807+0000", 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1364000000000002e-05 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.474 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:26498 11.f 11:f34ad7f0:::benchmark_data_smithi001_228075_object26497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:14:20.249547+0000", 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 230.19889537899999, 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18936424900000001, 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 26498 2024-09-14T04:18:10.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249547+0000", 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249547+0000", 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249546+0000", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249578+0000", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1619000000000003e-05 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249589+0000", 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1249e-05 2024-09-14T04:18:10.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.249596+0000", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4860000000000002e-06 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.250750+0000", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001154265 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.250871+0000", 2024-09-14T04:18:10.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012129300000000001 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.251001+0000", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013038500000000001 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.251958+0000", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00095687699999999997 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.253629+0000", 2024-09-14T04:18:10.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016704700000000001 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438813+0000", 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.18518442399999999 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438866+0000", 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2460000000000003e-05 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438911+0000", 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5618000000000003e-05 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:26519 11.5 11:ac640d2d:::benchmark_data_smithi001_228075_object26518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:14:20.259195+0000", 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 230.18924767199999, 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.187266401, 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 26519 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259195+0000", 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259195+0000", 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259193+0000", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259332+0000", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013883600000000001 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438775+0000", 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.179443253 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438788+0000", 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2768999999999999e-05 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438983+0000", 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000195136 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.439133+0000", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014960799999999999 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.439284+0000", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015158200000000001 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.443426+0000", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041416939999999996 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.443469+0000", 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2985999999999998e-05 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.446382+0000", 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029132170000000001 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.446428+0000", 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5639000000000001e-05 2024-09-14T04:18:10.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.446461+0000", 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3457000000000002e-05 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35451.0:26516 11.3 e248/244 11:cedae212:::benchmark_data_smithi001_228075_object26515:head v 248'1626, mlcod=248'1623)", 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:14:20.259395+0000", 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 230.18904746000001, 2024-09-14T04:18:10.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.182671417, 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259395+0000", 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259395+0000", 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259394+0000", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.259419+0000", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5250999999999999e-05 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438742+0000", 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.179322275 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438758+0000", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6313000000000001e-05 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438826+0000", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8180000000000001e-05 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.438848+0000", 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1480000000000001e-05 2024-09-14T04:18:10.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.441991+0000", 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031429990000000001 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:14:20.442066+0000", 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5809000000000006e-05 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111749 11.f 11:fa92e749:::benchmark_data_smithi001_228075_object111748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 158.02736950799999, 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0160433040000001, 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111749 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421072+0000", 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421102+0000", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0614000000000002e-05 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421115+0000", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2445e-05 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421122+0000", 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7770000000000002e-06 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421176+0000", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3489000000000002e-05 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421337+0000", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016110899999999999 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421484+0000", 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147243 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435948+0000", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014464069 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435958+0000", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7450000000000007e-06 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437034+0000", 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.001076007 2024-09-14T04:18:10.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437116+0000", 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2194999999999995e-05 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111769 11.f 11:f6a7532d:::benchmark_data_smithi001_228075_object111768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 157.97991327400001, 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98091847099999996, 2024-09-14T04:18:10.498 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111769 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468528+0000", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468582+0000", 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4372000000000001e-05 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468594+0000", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.19e-05 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468600+0000", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9710000000000001e-06 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468652+0000", 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1994999999999999e-05 2024-09-14T04:18:10.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468663+0000", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437585+0000", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96892259300000005 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437788+0000", 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000203069 2024-09-14T04:18:10.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437921+0000", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013228200000000001 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.442200+0000", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042791950000000004 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449285+0000", 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070849850000000002 2024-09-14T04:18:10.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449347+0000", 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1766000000000005e-05 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449448+0000", 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010103 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111770 11.4 11:2369255e:::benchmark_data_smithi001_228075_object111769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 157.97771322599999, 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97033377499999995, 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111770 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470728+0000", 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470785+0000", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6273999999999997e-05 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470795+0000", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0738e-05 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470801+0000", 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1869999999999996e-06 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470850+0000", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9265999999999999e-05 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470857+0000", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8770000000000004e-06 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433741+0000", 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96288411200000001 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433867+0000", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125864 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433977+0000", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010985 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.439362+0000", 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053849340000000001 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440897+0000", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015350260000000001 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440984+0000", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7182000000000007e-05 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441063+0000", 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9246999999999995e-05 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.510 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117402 11.3 11:c184ceb7:::benchmark_data_smithi001_228075_object117401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.89955308399999, 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.27739556599999998, 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.511 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117402 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548888+0000", 2024-09-14T04:18:10.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548957+0000", 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8973999999999994e-05 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548974+0000", 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7041000000000002e-05 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548982+0000", 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8639999999999996e-06 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.550732+0000", 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017508090000000001 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.550945+0000", 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000212924 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.551141+0000", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019523399999999999 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.555822+0000", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004681485 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.561382+0000", 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005560031 2024-09-14T04:18:10.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823453+0000", 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26207078 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823510+0000", 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6733e-05 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.826285+0000", 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027753119999999998 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117407 11.5 11:aca3a4ce:::benchmark_data_smithi001_228075_object117406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.89177810300001, 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26693572100000001, 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117407 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556663+0000", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557270+0000", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00060745000000000005 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557287+0000", 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6569999999999999e-05 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557294+0000", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4340000000000003e-06 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557341+0000", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6440000000000003e-05 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557525+0000", 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018454499999999999 2024-09-14T04:18:10.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557650+0000", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012483500000000001 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.564719+0000", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070693259999999999 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.571495+0000", 2024-09-14T04:18:10.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067759969999999997 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823510+0000", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.252014234 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823557+0000", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7543999999999997e-05 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823600+0000", 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2947000000000001e-05 2024-09-14T04:18:10.522 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117429 11.3 11:c5090e60:::benchmark_data_smithi001_228075_object117428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.873842013, 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.294253918, 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117429 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574599+0000", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574640+0000", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0890000000000003e-05 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574657+0000", 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7133999999999999e-05 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574664+0000", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5700000000000004e-06 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574733+0000", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8610999999999998e-05 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574937+0000", 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020398599999999999 2024-09-14T04:18:10.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.575163+0000", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000226166 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579049+0000", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038853379999999999 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.580021+0000", 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00097227100000000003 2024-09-14T04:18:10.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.865872+0000", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28585137599999999 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.865934+0000", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2028999999999999e-05 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868854+0000", 2024-09-14T04:18:10.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029201040000000002 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117431 11.5 11:a2ff9faa:::benchmark_data_smithi001_228075_object117430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.579070+0000", 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.869372664, 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.25695146600000002, 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117431 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579070+0000", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579070+0000", 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579068+0000", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579131+0000", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2811000000000006e-05 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579146+0000", 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4609e-05 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579155+0000", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2920000000000001e-06 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579209+0000", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3566000000000002e-05 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579368+0000", 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015914999999999999 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579522+0000", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015463599999999999 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.582856+0000", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033339630000000001 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.583998+0000", 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011419609999999999 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.835938+0000", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.25194019499999998 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.835983+0000", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5006000000000002e-05 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836021+0000", 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7798e-05 2024-09-14T04:18:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117433 11.5 11:a7c979d0:::benchmark_data_smithi001_228075_object117432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.585258+0000", 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.86318459099999, 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.25087331299999999, 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117433 2024-09-14T04:18:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585258+0000", 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585258+0000", 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585256+0000", 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585321+0000", 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5029000000000004e-05 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585336+0000", 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4358e-05 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585343+0000", 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5020000000000003e-06 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585391+0000", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7636999999999998e-05 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585508+0000", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116761 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.585609+0000", 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010100899999999999 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.589029+0000", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034207109999999999 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.589437+0000", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040716599999999998 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836057+0000", 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.24662028899999999 2024-09-14T04:18:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836097+0000", 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0011e-05 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836131+0000", 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4372999999999998e-05 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117444 11.5 11:aa87d517:::benchmark_data_smithi001_228075_object117443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.596843+0000", 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.851598986, 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.239403383, 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117444 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596843+0000", 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596843+0000", 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596839+0000", 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999957 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596888+0000", 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8783999999999999e-05 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596905+0000", 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7155000000000001e-05 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596914+0000", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5290000000000008e-06 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.596969+0000", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5751999999999997e-05 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.597121+0000", 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152029 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.597274+0000", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015262199999999999 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.600500+0000", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032258550000000001 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601495+0000", 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099473500000000006 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836170+0000", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.23467516799999999 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836210+0000", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0411000000000003e-05 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.836247+0000", 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6745999999999997e-05 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117447 11.3 11:c9a66c9a:::benchmark_data_smithi001_228075_object117446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.84699319999999, 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26974246400000002, 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117447 2024-09-14T04:18:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601448+0000", 2024-09-14T04:18:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601496+0000", 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8690000000000003e-05 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601509+0000", 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2318e-05 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601515+0000", 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.781e-06 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601562+0000", 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6851000000000003e-05 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601707+0000", 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014509899999999999 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601815+0000", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107524 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.604811+0000", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029959470000000001 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.605687+0000", 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087606800000000003 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868906+0000", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26321921999999998 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868958+0000", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2215000000000001e-05 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.871192+0000", 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022335160000000001 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.554 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117458 11.3 11:cc157f1c:::benchmark_data_smithi001_228075_object117457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.632852+0000", 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 150.81559071999999, 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.24026498499999999, 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117458 2024-09-14T04:18:10.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632852+0000", 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632852+0000", 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632851+0000", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632888+0000", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7576000000000001e-05 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632898+0000", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9459999999999993e-06 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632903+0000", 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1340000000000003e-06 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.632958+0000", 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4435999999999999e-05 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.633127+0000", 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016935600000000001 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.633239+0000", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111749 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.636177+0000", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002938324 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.636636+0000", 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000459073 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.871253+0000", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.23461639300000001 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.871310+0000", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7623e-05 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.873117+0000", 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018062919999999999 2024-09-14T04:18:10.560 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:169646 11.3 11:c0af481a:::benchmark_data_smithi001_228075_object51331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:16:33.277252+0000", 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.171189982000001, 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.111270963, 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.561 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 169646 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277252+0000", 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277252+0000", 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277251+0000", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277257+0000", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0270000000000001e-06 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277266+0000", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2800000000000003e-06 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.277273+0000", 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.294395+0000", 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017122024 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for rw locks", 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.294532+0000", 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137164 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.358460+0000", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063927625000000002 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.358548+0000", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8654999999999997e-05 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.358692+0000", 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014342200000000001 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.373789+0000", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015097289999999999 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.373892+0000", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010343 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.387165+0000", 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013272644 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.387500+0000", 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00033504699999999999 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:33.388523+0000", 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010234179999999999 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.567 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:173542 11.3 11:c363f904:::benchmark_data_smithi001_228075_object55227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:16:37.646469+0000", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 92.801973861999997, 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.106919874, 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 173542 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646469+0000", 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646469+0000", 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646468+0000", 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646473+0000", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9230000000000001e-06 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646483+0000", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0309000000000001e-05 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.646490+0000", 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.562e-06 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.664355+0000", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017865853000000001 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for rw locks", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.664570+0000", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000214952 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.707862+0000", 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043292123000000002 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.707957+0000", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4970999999999996e-05 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.708118+0000", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016098499999999999 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.729979+0000", 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021861023 2024-09-14T04:18:10.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.731745+0000", 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001765375 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.752136+0000", 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020391366000000001 2024-09-14T04:18:10.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.752188+0000", 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2126000000000002e-05 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:37.753388+0000", 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012001850000000001 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.574 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:186555 11.3 11:cd4f7138:::benchmark_data_smithi001_228075_object68240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:16:53.812550+0000", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 76.635892381000005, 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082428989999999994, 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 186555 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812550+0000", 2024-09-14T04:18:10.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812550+0000", 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812549+0000", 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812555+0000", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2380000000000002e-06 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812566+0000", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0771000000000001e-05 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.812573+0000", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3989999999999998e-06 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.817185+0000", 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046123070000000004 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.817308+0000", 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012219499999999999 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.817437+0000", 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129417 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.821990+0000", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045527900000000001 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.822462+0000", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047226700000000001 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.855042+0000", 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032579785 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.894046+0000", 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039004490000000003 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:16:53.894979+0000", 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00093266200000000001 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.580 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:212110 11.3 11:c68a7eca:::benchmark_data_smithi001_228075_object93795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:21.634213+0000", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 48.814229861999998, 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.075424273999999999, 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212110 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634213+0000", 2024-09-14T04:18:10.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634213+0000", 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634211+0000", 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634215+0000", 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2169999999999996e-06 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634223+0000", 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0229999999999992e-06 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.634229+0000", 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3170000000000001e-06 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.696297+0000", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062068672999999998 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.696424+0000", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001261 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.696537+0000", 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011382 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.705977+0000", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009439958 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.707879+0000", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019016899999999999 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.709118+0000", 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001239048 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.709169+0000", 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0784e-05 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.709637+0000", 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046806499999999998 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35451.0:212118 11.0 e274/244 11:00fe2c06:::benchmark_data_smithi001_228075_object93803:head v 274'13371, mlcod=274'13368)", 2024-09-14T04:18:10.586 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:17:21.645820+0000", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 48.802622202000002, 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.081448791000000006, 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.645820+0000", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.645820+0000", 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:18:10.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.645820+0000", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.645825+0000", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4070000000000001e-06 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.694867+0000", 2024-09-14T04:18:10.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049042232999999998 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.694884+0000", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6636000000000001e-05 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.695017+0000", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013340199999999999 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.695049+0000", 2024-09-14T04:18:10.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1333e-05 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.727237+0000", 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032188693999999997 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:17:21.727269+0000", 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1776999999999997e-05 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:18:10.590 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:18:10.591 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:18:10.591 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:10 smithi012 ceph-mon[101180]: pgmap v1021: 99 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:10.591 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:10 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:18:10.591 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:10 smithi012 ceph-mon[101180]: osdmap e284: 8 total, 7 up, 8 in 2024-09-14T04:18:10.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:10.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[107399]: pgmap v1021: 99 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:10.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:18:10.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[107399]: osdmap e284: 8 total, 7 up, 8 in 2024-09-14T04:18:10.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[112165]: pgmap v1021: 99 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 59 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:10.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:18:10.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:10 smithi001 ceph-mon[112165]: osdmap e284: 8 total, 7 up, 8 in 2024-09-14T04:18:10.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:10.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:10.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:10.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:10.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:10.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:10.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:11.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:11.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:11.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:11.246 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:11.264 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:18:11.264 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:18:11.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-14T04:18:11.304 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:11.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:11.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:11.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:11.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:11.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.498 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:11.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:11.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:18:11.532 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:11 smithi001 ceph-mon[107399]: osdmap e285: 8 total, 7 up, 8 in 2024-09-14T04:18:11.533 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:11 smithi001 ceph-mon[112165]: osdmap e285: 8 total, 7 up, 8 in 2024-09-14T04:18:11.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:11.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:11.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:11.746 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:11.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:11 smithi012 ceph-mon[101180]: osdmap e285: 8 total, 7 up, 8 in 2024-09-14T04:18:11.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:11.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:11.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:11.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:11.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:12.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:12.123 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:12.224 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:12.248 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:12.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:12.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:12.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:12.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:12.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:12.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:12.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:12.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:12 smithi001 ceph-mon[107399]: pgmap v1024: 99 pgs: 5 creating+peering, 5 unknown, 3 active+clean+remapped, 22 active+undersized, 3 active+undersized+degraded, 61 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:12 smithi001 ceph-mon[112165]: pgmap v1024: 99 pgs: 5 creating+peering, 5 unknown, 3 active+clean+remapped, 22 active+undersized, 3 active+undersized+degraded, 61 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:12.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:12.755 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:12 smithi012 ceph-mon[101180]: pgmap v1024: 99 pgs: 5 creating+peering, 5 unknown, 3 active+clean+remapped, 22 active+undersized, 3 active+undersized+degraded, 61 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:12.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:12.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:12.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:12.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:18:12.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:13.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:13.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:13.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:13.232 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:13.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:13.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:13.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:13.481 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:13.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:13.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:13.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.711 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:13.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:13.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:13.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:13.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:13.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:14.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:14.030 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:14.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:14.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:14.235 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:14.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:14.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:14.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:14.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:14.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:14.531 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[107399]: pgmap v1025: 99 pgs: 5 creating+peering, 3 active+clean+remapped, 25 active+undersized, 3 active+undersized+degraded, 63 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:14.532 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3493394512' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.532 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.532 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[112165]: pgmap v1025: 99 pgs: 5 creating+peering, 3 active+clean+remapped, 25 active+undersized, 3 active+undersized+degraded, 63 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:14.532 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3493394512' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.532 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:14 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:14 smithi012 ceph-mon[101180]: pgmap v1025: 99 pgs: 5 creating+peering, 3 active+clean+remapped, 25 active+undersized, 3 active+undersized+degraded, 63 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3493394512' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:14 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:14.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:14.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:14.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:14.746 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:18:14.747 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:18:14.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:14.763 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:14.764 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:18:14.764 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:18:14.765 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:18:14.782 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:18:14.782 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:18:14.799 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:18:14.799 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:18:14.816 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:18:14.816 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:18:14.832 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:18:14.833 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:18:14.848 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:18:14.848 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:18:14.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:14.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:14.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:14.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:15.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:15.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:15.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:15.146 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_7' 2024-09-14T04:18:15.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:15.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:15.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:15.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:15.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2716413768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[107399]: osdmap e286: 8 total, 7 up, 8 in 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2716413768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:15.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:15 smithi001 ceph-mon[112165]: osdmap e286: 8 total, 7 up, 8 in 2024-09-14T04:18:15.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:15.507 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2716413768' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:15 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:15.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:15 smithi012 ceph-mon[101180]: osdmap e286: 8 total, 7 up, 8 in 2024-09-14T04:18:15.609 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:15.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:15.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:15.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:18:15.806 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:18:15.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:15.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:15.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:15.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:16.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:16.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:16.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:16.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:16.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:16.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:16.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:16.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:16.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:16.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:16.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:16.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:16.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:16 smithi001 ceph-mon[107399]: pgmap v1027: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:16.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:16 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:18:16.680 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:16 smithi001 ceph-mon[112165]: pgmap v1027: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:16.680 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:16 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:18:16.689 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:16 smithi012 ceph-mon[101180]: pgmap v1027: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:16.689 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:16 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:18:16.731 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:16.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:16.855 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:16.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:16.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:16.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:16] "GET /metrics HTTP/1.1" 200 34211 "" "Prometheus/2.51.0" 2024-09-14T04:18:16.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:17.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:17.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:17.128 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:17.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:17.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:17.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:17.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:17.471 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2024-09-14T04:18:17.474 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:18:17.475 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:18:17.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:17.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:17.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:18:17.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:18:17.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_255951 2024-09-14T04:18:17.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:18:17.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:18:17.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:17.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:17.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:17.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:17.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:18:17.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:17.913 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:17.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:18.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:18.050 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:18.153 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:18.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:18.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:18.330 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.330 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.331 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[112165]: pgmap v1028: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:18.331 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.331 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.331 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:18 smithi001 ceph-mon[107399]: pgmap v1028: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:18.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:18 smithi012 ceph-mon[101180]: pgmap v1028: 99 pgs: 3 active+clean+remapped, 27 active+undersized, 3 active+undersized+degraded, 66 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-14T04:18:18.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:18.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:18.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:18.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:18.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 4258 4242 265.109 265.125 0.00376986 0.00375569 2024-09-14T04:18:18.588 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:18.630 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:18.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:18.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:18.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:18.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:18.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:18.911 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:18:18.911 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:18:19.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:19.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:19.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:19.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:19.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:19.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:19.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:19.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:19.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 8254 8238 257.418 249.75 0.00197681 0.00387676 2024-09-14T04:18:19.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:19.587 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:19.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 803/6042 objects degraded (13.290%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 803/6042 objects degraded (13.290%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:19.724 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:19.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 803/6042 objects degraded (13.290%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:19.864 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:19.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:19.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:20.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:20.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:20.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.271 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:20.373 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:20.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.529 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 11680 11664 242.979 214.125 0.00204082 0.0041065 2024-09-14T04:18:20.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:20.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[107399]: pgmap v1029: 99 pgs: 3 active+clean+remapped, 25 active+undersized, 5 active+undersized+degraded, 66 active+clean; 125 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s; 803/6042 objects degraded (13.290%) 2024-09-14T04:18:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[112165]: pgmap v1029: 99 pgs: 3 active+clean+remapped, 25 active+undersized, 5 active+undersized+degraded, 66 active+clean; 125 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s; 803/6042 objects degraded (13.290%) 2024-09-14T04:18:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:20.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:20.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:20 smithi012 ceph-mon[101180]: pgmap v1029: 99 pgs: 3 active+clean+remapped, 25 active+undersized, 5 active+undersized+degraded, 66 active+clean; 125 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s; 803/6042 objects degraded (13.290%) 2024-09-14T04:18:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:20.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:20.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:20.949 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:20.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:no valid command found; 10 closest matches: 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:0 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:1 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:2 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:abort 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:assert 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:bench [] [] [] [] 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs files list 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:bluefs stats 2024-09-14T04:18:21.015 INFO:teuthology.orchestra.run.smithi001.stderr:bluestore allocator dump block 2024-09-14T04:18:21.016 INFO:teuthology.orchestra.run.smithi001.stderr:Error EINVAL: invalid command 2024-09-14T04:18:21.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:21.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:21.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:21.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:21.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:21.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:21.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:21.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:21.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:21.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:21.530 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 14676 14660 229.043 187.25 0.00370489 0.00436021 2024-09-14T04:18:21.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:21.641 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:21.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:21.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:21.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:21.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:21.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:21.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:22.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:22.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:22.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:22.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:22.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:22.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:22.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:22.530 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 18200 18184 227.28 220.25 0.00370605 0.00439393 2024-09-14T04:18:22.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:22.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:22.691 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:22 smithi001 ceph-mon[112165]: pgmap v1030: 99 pgs: 3 active+clean+remapped, 21 active+undersized, 9 active+undersized+degraded, 66 active+clean; 438 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 4137/21069 objects degraded (19.635%) 2024-09-14T04:18:22.691 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:22 smithi001 ceph-mon[107399]: pgmap v1030: 99 pgs: 3 active+clean+remapped, 21 active+undersized, 9 active+undersized+degraded, 66 active+clean; 438 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 4137/21069 objects degraded (19.635%) 2024-09-14T04:18:22.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:22.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:22 smithi012 ceph-mon[101180]: pgmap v1030: 99 pgs: 3 active+clean+remapped, 21 active+undersized, 9 active+undersized+degraded, 66 active+clean; 438 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 4137/21069 objects degraded (19.635%) 2024-09-14T04:18:22.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:22.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:22.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:18:22.900 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T04:18:22.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:23.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:23.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:23.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:23.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:23.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:23.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:23.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:18:23.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:23.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:23.530 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 21441 21425 223.158 202.562 0.00406686 0.00447581 2024-09-14T04:18:23.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:23.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:23.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:23.726 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:23.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:23.847 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:23.847 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:23.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:23.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:24.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:24.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:24.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:24.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:24.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:24.254 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:24.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:24.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:24.427 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:24.464 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:24.530 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 24341 24325 217.147 181.25 0.00542429 0.0045992 2024-09-14T04:18:24.566 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:24.603 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:24.605 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:24 smithi001 ceph-mon[112165]: pgmap v1031: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 753 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 7069/36171 objects degraded (19.543%) 2024-09-14T04:18:24.605 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:24 smithi001 ceph-mon[107399]: pgmap v1031: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 753 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 7069/36171 objects degraded (19.543%) 2024-09-14T04:18:24.705 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:24.740 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:24.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:24 smithi012 ceph-mon[101180]: pgmap v1031: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 753 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 7069/36171 objects degraded (19.543%) 2024-09-14T04:18:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:24.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:24.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:24.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:25.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:25.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:25.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:25.153 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:25.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:25.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:25.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:25.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:25.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:25.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 15 27348 27333 213.502 188 0.00411827 0.00467895 2024-09-14T04:18:25.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:25.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:25.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:25.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:25.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:25 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9111/51729 objects degraded (17.613%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:25 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9111/51729 objects degraded (17.613%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:25.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:25 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9111/51729 objects degraded (17.613%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:25.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:25.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:25.963 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:26.000 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:26.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:26.136 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:26.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:26.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:26.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:26.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:26.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:26.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 29869 29853 207.277 157.5 0.00370111 0.00481959 2024-09-14T04:18:26.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:26.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:26.657 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:26.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:26.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:26.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:26.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:26 smithi012 ceph-mon[101180]: pgmap v1032: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 9111/51729 objects degraded (17.613%) 2024-09-14T04:18:26.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:26.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:26.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:26 smithi001 ceph-mon[107399]: pgmap v1032: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 9111/51729 objects degraded (17.613%) 2024-09-14T04:18:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:26 smithi001 ceph-mon[112165]: pgmap v1032: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 9111/51729 objects degraded (17.613%) 2024-09-14T04:18:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:26] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:18:27.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:27.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:27.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:27.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:27.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:27.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:27.362 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:27.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:27.497 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:27.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 31979 31963 199.736 131.875 0.00194304 0.0049863 2024-09-14T04:18:27.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:27.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:27.729 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.752 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:27.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:27.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:27.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:27.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:27.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:27.901 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2024-09-14T04:18:27.901 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:18:27.901 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:18:27.901 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2024-09-14T04:18:27.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-14T04:18:27.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:28.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:28.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:28.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:28.159 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:28.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:28.283 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:28.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:28.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:28.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:28.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 33742 33726 191.595 110.188 0.00942931 0.0052143 2024-09-14T04:18:28.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:28.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:28.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:28.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:28.788 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:28.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:28 smithi012 ceph-mon[101180]: pgmap v1033: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 15656/82854 objects degraded (18.896%) 2024-09-14T04:18:28.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:28.949 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:28.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:28 smithi001 ceph-mon[107399]: pgmap v1033: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 15656/82854 objects degraded (18.896%) 2024-09-14T04:18:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:28 smithi001 ceph-mon[112165]: pgmap v1033: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 15656/82854 objects degraded (18.896%) 2024-09-14T04:18:29.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:29.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:29.190 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:29.222 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:29.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:29.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:29.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:29.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:29.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:29.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 35934 35918 187.045 137 0.00614972 0.00534137 2024-09-14T04:18:29.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:29.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:29.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:29.717 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:29.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:29.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:29.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:29.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:30.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:30.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:30.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:30.144 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:30.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:30.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:30.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:30.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:30.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:30.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:30.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 38224 38208 183.665 143.125 0.0105397 0.00544058 2024-09-14T04:18:30.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:30.599 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:30.701 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:30.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:30.771 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:30 smithi001 ceph-mon[112165]: pgmap v1034: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 17144/94071 objects degraded (18.225%) 2024-09-14T04:18:30.771 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:30 smithi001 ceph-mon[107399]: pgmap v1034: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 17144/94071 objects degraded (18.225%) 2024-09-14T04:18:30.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:30 smithi012 ceph-mon[101180]: pgmap v1034: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 17144/94071 objects degraded (18.225%) 2024-09-14T04:18:30.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:30.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:30.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:31.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:31.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:31.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:31.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:31.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:31.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:31.430 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:31.478 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:18:31.478 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:18:31.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 40546 40530 180.912 145.125 0.004503 0.00552341 2024-09-14T04:18:31.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:31.575 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:31 smithi001 ceph-mon[112165]: pgmap v1035: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s; 19786/105936 objects degraded (18.677%) 2024-09-14T04:18:31.575 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:31 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 19786/105936 objects degraded (18.677%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:31.575 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:31 smithi001 ceph-mon[107399]: pgmap v1035: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s; 19786/105936 objects degraded (18.677%) 2024-09-14T04:18:31.575 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:31 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 19786/105936 objects degraded (18.677%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:31.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:31.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:31.811 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:31 smithi012 ceph-mon[101180]: pgmap v1035: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s; 19786/105936 objects degraded (18.677%) 2024-09-14T04:18:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:31 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 19786/105936 objects degraded (18.677%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:31.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:31.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:31.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:31.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:31.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:32.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:32.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.131 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:32.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:32.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:32.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:32.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:32.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:32.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:32.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:32.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 42890 42874 178.617 146.5 0.00932093 0.0055937 2024-09-14T04:18:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:32.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:32.605 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:18:32.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:32.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:18:32.679 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:18:32.683 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:32.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:32.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:18:32.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:32.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:32.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:32.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:32.959 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:32.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:33.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:18:33.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:33.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:33.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:33.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:33.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:33.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:33.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:33.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:33.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 45150 45134 176.281 141.25 0.00439912 0.00566795 2024-09-14T04:18:33.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.656 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:33.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:33.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:33.789 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:33.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:33.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:33.894 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:33.919 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:33.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:33.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[107399]: osdmap e287: 8 total, 7 up, 8 in 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[107399]: pgmap v1037: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s; 21848/116208 objects degraded (18.801%) 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:18:33.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[112165]: osdmap e287: 8 total, 7 up, 8 in 2024-09-14T04:18:33.974 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:33 smithi001 ceph-mon[112165]: pgmap v1037: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s; 21848/116208 objects degraded (18.801%) 2024-09-14T04:18:34.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:34.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:33 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:33 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/960932800' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:18:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:33 smithi012 ceph-mon[101180]: osdmap e287: 8 total, 7 up, 8 in 2024-09-14T04:18:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:33 smithi012 ceph-mon[101180]: pgmap v1037: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s; 21848/116208 objects degraded (18.801%) 2024-09-14T04:18:34.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:34.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:34.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:34.341 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:34.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:34.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:34.514 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:34.531 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 47229 47213 173.554 129.938 0.0147795 0.00575591 2024-09-14T04:18:34.585 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:34.590 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:18:34.590 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:18:34.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:34.626 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:18:34.627 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:18:34.632 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:18:34.664 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:34 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1275563340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:34.665 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:18:34.665 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:18:34.665 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:34 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1275563340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:34.697 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:18:34.697 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:18:34.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:34.735 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:18:34.735 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:18:34.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:34.777 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:18:34.777 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:18:34.823 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:18:34.823 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:18:34.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:34.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:34.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:34.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:35.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:35.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:34 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1275563340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:18:35.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:35.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:35.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:35.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:35.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:35.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:18:35.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:35.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:35.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 15 49301 49286 171.109 129.562 0.0110907 0.00583883 2024-09-14T04:18:35.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:35.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:35.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:35.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:35 smithi001 ceph-mon[107399]: pgmap v1038: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s; 23368/127236 objects degraded (18.366%) 2024-09-14T04:18:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:35.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:35 smithi001 ceph-mon[112165]: pgmap v1038: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s; 23368/127236 objects degraded (18.366%) 2024-09-14T04:18:35.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:35.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:35.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:35.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:35.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:35.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:36.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:35 smithi012 ceph-mon[101180]: pgmap v1038: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s; 23368/127236 objects degraded (18.366%) 2024-09-14T04:18:36.120 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:36.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:36.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:36.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.294 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:36.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:36.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:36.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:36.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 51387 51371 168.96 130.312 0.00304313 0.00591403 2024-09-14T04:18:36.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:36.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:36.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:36.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:36.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:36.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:36.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:36] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:18:36.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:37.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:18:37.060 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:18:37.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:37.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:37.268 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:37.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:37.407 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:37.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:37.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:18:37.531974+0000 min lat: 0.000960706 max lat: 0.159472 avg lat: 0.00597946 2024-09-14T04:18:37.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:18:37.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 53496 53480 167.102 131.812 0.0150413 0.00597946 2024-09-14T04:18:37.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:37.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:37.604 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:37 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 28013/149193 objects degraded (18.776%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:37.604 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:37 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 28013/149193 objects degraded (18.776%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:37.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:37.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:37.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:37.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:37 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 28013/149193 objects degraded (18.776%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:37.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:37.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:37.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:38.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:38.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:38.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:38.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:38.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:38.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:38.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:38.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:38.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:38.527 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:38 smithi001 ceph-mon[112165]: pgmap v1039: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 28013/149193 objects degraded (18.776%) 2024-09-14T04:18:38.527 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:38 smithi001 ceph-mon[107399]: pgmap v1039: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 28013/149193 objects degraded (18.776%) 2024-09-14T04:18:38.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 55372 55356 164.728 117.25 0.00450376 0.00606567 2024-09-14T04:18:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.614 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:38.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:38.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:38.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:38.767 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:38 smithi012 ceph-mon[101180]: pgmap v1039: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 28013/149193 objects degraded (18.776%) 2024-09-14T04:18:38.855 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:190059 10.8 10:162cc157:::benchmark_data_smithi001_190881_object74919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:10.621038+0000", 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 388.22905916899998, 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.32226967499999998, 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.856 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 190059 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621038+0000", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621038+0000", 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621040+0000", 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8279999999999999e-06 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621045+0000", 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4199999999999998e-06 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621058+0000", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2444e-05 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621065+0000", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3250000000000002e-06 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621166+0000", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000100771 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621269+0000", 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103368 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.621374+0000", 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010450099999999999 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.629055+0000", 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007681467 2024-09-14T04:18:38.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.629067+0000", 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1647999999999999e-05 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.943204+0000", 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.31413683199999998 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.943260+0000", 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6385999999999999e-05 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:10.943308+0000", 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7685000000000002e-05 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111750 11.6 11:6780736c:::benchmark_data_smithi001_228075_object111749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 186.422450126, 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0279308629999999, 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111750 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427649+0000", 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8339999999999999e-06 2024-09-14T04:18:38.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427674+0000", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4975000000000001e-05 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427687+0000", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3276e-05 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427697+0000", 2024-09-14T04:18:38.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3239999999999992e-06 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431409+0000", 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037127940000000002 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431421+0000", 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1535000000000001e-05 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438346+0000", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.006924989 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438476+0000", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000129815 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438613+0000", 2024-09-14T04:18:38.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013711100000000001 2024-09-14T04:18:38.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447342+0000", 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087294480000000008 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455412+0000", 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080694630000000007 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455479+0000", 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7631000000000004e-05 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455578+0000", 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8667999999999998e-05 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111764 11.6 11:66131855:::benchmark_data_smithi001_228075_object111763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 186.39539945499999, 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0049983549999999, 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.871 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111764 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454700+0000", 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.046e-06 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454723+0000", 2024-09-14T04:18:38.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3271000000000001e-05 2024-09-14T04:18:38.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454741+0000", 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8281999999999999e-05 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454749+0000", 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4370000000000001e-06 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454812+0000", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3596000000000006e-05 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454823+0000", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1090000000000001e-05 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.442746+0000", 2024-09-14T04:18:38.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98792279299999997 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.442898+0000", 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015215300000000001 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.443068+0000", 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000169958 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455636+0000", 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012567807 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459433+0000", 2024-09-14T04:18:38.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037966279999999998 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459607+0000", 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017445899999999999 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459696+0000", 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8834999999999999e-05 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111766 11.6 11:61ce0161:::benchmark_data_smithi001_228075_object111765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 186.39413149200001, 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0098253399999999, 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.881 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111766 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455967+0000", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1629999999999999e-06 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455988+0000", 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1546999999999999e-05 2024-09-14T04:18:38.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456002+0000", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3473e-05 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456008+0000", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8939999999999998e-06 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456061+0000", 2024-09-14T04:18:38.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2991999999999997e-05 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456071+0000", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0327e-05 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447464+0000", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.991392618 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447619+0000", 2024-09-14T04:18:38.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000154878 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447781+0000", 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000162826 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459705+0000", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011923173 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465640+0000", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059350999999999996 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465701+0000", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0872000000000003e-05 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465791+0000", 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0476999999999994e-05 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:48637 12.6 12:63cdd892:::benchmark_data_smithi001_255951_object48636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:35.214292+0000", 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 3.6358055920000001, 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019473972999999999, 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.888 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.889 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:38.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.890 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 48637 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214292+0000", 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214292+0000", 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214293+0000", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3170000000000001e-06 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214341+0000", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8075000000000003e-05 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214354+0000", 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2921e-05 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214360+0000", 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5429999999999999e-06 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.217350+0000", 2024-09-14T04:18:38.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029890429999999998 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.217456+0000", 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010679399999999999 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.217578+0000", 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000121761 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.221497+0000", 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039187960000000004 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.221506+0000", 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3549999999999994e-06 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233663+0000", 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012156975 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233724+0000", 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.122e-05 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233766+0000", 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1173e-05 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:48638 12.e 12:7c21299c:::benchmark_data_smithi001_255951_object48637:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:35.214897+0000", 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 3.6352003050000001, 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019028100999999999, 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.897 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 48638 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214897+0000", 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214897+0000", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214898+0000", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5300000000000002e-07 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214942+0000", 2024-09-14T04:18:38.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4122999999999998e-05 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214953+0000", 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1134e-05 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.214959+0000", 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4349999999999996e-06 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.219377+0000", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044188329999999996 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.219654+0000", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027617599999999999 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.219789+0000", 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135821 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.224696+0000", 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0049070149999999998 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.224857+0000", 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000160533 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233838+0000", 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089809590000000002 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233892+0000", 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3783000000000003e-05 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.233925+0000", 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3336000000000001e-05 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:49034 12.e 12:7433136e:::benchmark_data_smithi001_255951_object49033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:35.392500+0000", 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 3.4575972340000001, 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018940603, 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 49034 2024-09-14T04:18:38.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392500+0000", 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392500+0000", 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392500+0000", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9500000000000003e-07 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392520+0000", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9981999999999999e-05 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392531+0000", 2024-09-14T04:18:38.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0247e-05 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.392536+0000", 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1510000000000003e-06 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.394517+0000", 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0019813119999999998 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.395053+0000", 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00053605500000000002 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.395219+0000", 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016607199999999999 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.408100+0000", 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012880832 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.408138+0000", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8114999999999997e-05 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.411348+0000", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032100850000000001 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.411401+0000", 2024-09-14T04:18:38.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2484000000000001e-05 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.411441+0000", 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9773e-05 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:49690 12.6 12:625eb382:::benchmark_data_smithi001_255951_object49689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:35.725556+0000", 2024-09-14T04:18:38.910 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 3.1245411679999999, 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019419331000000001, 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 49690 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725556+0000", 2024-09-14T04:18:38.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725556+0000", 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725557+0000", 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9700000000000005e-07 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725577+0000", 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0105000000000001e-05 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725592+0000", 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4636e-05 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.725600+0000", 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4209999999999995e-06 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.727178+0000", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015782509999999999 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.727305+0000", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012670499999999999 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.727439+0000", 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000133456 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.733513+0000", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0060747129999999998 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.735955+0000", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024420119999999999 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.744852+0000", 2024-09-14T04:18:38.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088969759999999991 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.744935+0000", 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2953000000000004e-05 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:35.744975+0000", 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0206000000000001e-05 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:51176 12.6 12:63ea0dbe:::benchmark_data_smithi001_255951_object51175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:36.422694+0000", 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.4274029189999999, 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018944742000000001, 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 51176 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422694+0000", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422694+0000", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422695+0000", 2024-09-14T04:18:38.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7499999999999999e-07 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422717+0000", 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1687000000000001e-05 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422728+0000", 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1382999999999999e-05 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.422735+0000", 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9650000000000002e-06 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.428297+0000", 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0055621079999999996 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.428445+0000", 2024-09-14T04:18:38.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000148269 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.428601+0000", 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015555899999999999 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.433043+0000", 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044417340000000001 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.435518+0000", 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002475451 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.441530+0000", 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0060114440000000003 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.441582+0000", 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2079000000000001e-05 2024-09-14T04:18:38.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.441639+0000", 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7287999999999998e-05 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:51445 12.6 12:6baaf0b4:::benchmark_data_smithi001_255951_object51444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:36.556494+0000", 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 2.2936029900000001, 2024-09-14T04:18:38.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019856279000000001, 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 51445 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556494+0000", 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556494+0000", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556495+0000", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999999e-07 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556519+0000", 2024-09-14T04:18:38.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3632999999999999e-05 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556530+0000", 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1129e-05 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.556536+0000", 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5860000000000001e-06 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.563600+0000", 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007063115 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.563703+0000", 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103528 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.563859+0000", 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000155823 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.571171+0000", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073118219999999999 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.573691+0000", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025198019999999998 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.576242+0000", 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025511589999999999 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.576308+0000", 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6760000000000005e-05 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:36.576351+0000", 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2046999999999999e-05 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.929 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:52461 12.e 12:710e74d3:::benchmark_data_smithi001_255951_object52460:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:37.035383+0000", 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 1.8147144879999999, 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019747271, 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 52461 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035383+0000", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035383+0000", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035384+0000", 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3990000000000001e-06 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035433+0000", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9106e-05 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035444+0000", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0417999999999999e-05 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035450+0000", 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1909999999999997e-06 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035502+0000", 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2448000000000003e-05 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035625+0000", 2024-09-14T04:18:38.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012226399999999999 2024-09-14T04:18:38.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.035755+0000", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000130808 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.039659+0000", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039035710000000002 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.040028+0000", 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00036932399999999998 2024-09-14T04:18:38.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.055037+0000", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015008339000000001 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.055088+0000", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1619000000000002e-05 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:37.055130+0000", 2024-09-14T04:18:38.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1783999999999998e-05 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:54922 12.6 12:6554fc27:::benchmark_data_smithi001_255951_object54921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.269822+0000", 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.58027478300000002, 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.01901452, 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 54922 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269822+0000", 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269822+0000", 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269824+0000", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.088e-06 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269859+0000", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4990000000000002e-05 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269867+0000", 2024-09-14T04:18:38.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6829999999999997e-06 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269872+0000", 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6469999999999997e-06 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269910+0000", 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7916000000000001e-05 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.269993+0000", 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2735e-05 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.270078+0000", 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5543000000000004e-05 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.271362+0000", 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001283979 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.274591+0000", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032285780000000002 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.288730+0000", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014139647 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.288789+0000", 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8671000000000001e-05 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.288837+0000", 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8043000000000001e-05 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55268 12.e 12:7c2a52d3:::benchmark_data_smithi001_255951_object55267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.473952+0000", 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.37614565799999999, 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019833793999999998, 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.944 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55268 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473952+0000", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.945 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473952+0000", 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473952+0000", 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8999999999999997e-07 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473973+0000", 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0581000000000002e-05 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.946 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473979+0000", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8469999999999998e-06 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473985+0000", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0309999999999998e-06 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.476093+0000", 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002108909 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.476369+0000", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027587199999999998 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.476508+0000", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001383 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.485404+0000", 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088963689999999995 2024-09-14T04:18:38.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.490694+0000", 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052896200000000001 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493696+0000", 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0030027040000000001 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493736+0000", 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9786000000000002e-05 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493785+0000", 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9285e-05 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.950 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55269 12.e 12:7de238b5:::benchmark_data_smithi001_255951_object55268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.473992+0000", 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.37610507799999998, 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019891485, 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55269 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473992+0000", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473992+0000", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.473993+0000", 2024-09-14T04:18:38.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4999999999999998e-07 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.474015+0000", 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2161e-05 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.474022+0000", 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7649999999999996e-06 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.474027+0000", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3299999999999998e-06 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.478485+0000", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044576650000000004 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.478595+0000", 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011039699999999999 2024-09-14T04:18:38.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.478715+0000", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120159 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.485453+0000", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006738211 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.490729+0000", 2024-09-14T04:18:38.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052755110000000001 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493795+0000", 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003066315 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493847+0000", 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1693999999999997e-05 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.493884+0000", 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6826999999999998e-05 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55274 12.e 12:7ee80c1c:::benchmark_data_smithi001_255951_object55273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.477097+0000", 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.37300029699999998, 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.023123044999999998, 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.957 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55274 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477097+0000", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.958 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477097+0000", 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477098+0000", 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0160000000000001e-06 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477123+0000", 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5188999999999999e-05 2024-09-14T04:18:38.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477136+0000", 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2658e-05 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.477143+0000", 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3669999999999999e-06 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.482864+0000", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057207350000000002 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.483011+0000", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014688599999999999 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.483156+0000", 2024-09-14T04:18:38.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000145055 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.488042+0000", 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0048859840000000003 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.490857+0000", 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028150430000000001 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500128+0000", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0092714709999999999 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500179+0000", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1133e-05 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500220+0000", 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0507999999999999e-05 2024-09-14T04:18:38.963 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55280 12.e 12:702280a4:::benchmark_data_smithi001_255951_object55279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.480202+0000", 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.36989572999999998, 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020099887, 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.964 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55280 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480202+0000", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480202+0000", 2024-09-14T04:18:38.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480203+0000", 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.55e-06 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480222+0000", 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9023999999999999e-05 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480238+0000", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5758999999999999e-05 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.480245+0000", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9630000000000001e-06 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.485484+0000", 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052388019999999999 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.485617+0000", 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000133697 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.485763+0000", 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014583900000000001 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.488082+0000", 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023189809999999999 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.490893+0000", 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028110240000000001 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500227+0000", 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0093343330000000002 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.969 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500269+0000", 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1424999999999997e-05 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.500301+0000", 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2490000000000002e-05 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.970 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55489 12.e 12:7fe83d6f:::benchmark_data_smithi001_255951_object55488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.602135+0000", 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.24796180500000001, 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020001663999999999, 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55489 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.971 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602135+0000", 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602135+0000", 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602137+0000", 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6300000000000001e-06 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602171+0000", 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3448000000000002e-05 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602184+0000", 2024-09-14T04:18:38.973 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3896e-05 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602193+0000", 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0420000000000008e-06 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602244+0000", 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0810999999999999e-05 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.974 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602387+0000", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014258200000000001 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602549+0000", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016256699999999999 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.605754+0000", 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032045519999999998 2024-09-14T04:18:38.975 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.611108+0000", 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053538229999999997 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.622020+0000", 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010912720000000001 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.622093+0000", 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2885000000000004e-05 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.622137+0000", 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3708000000000002e-05 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.977 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55490 12.e 12:709a43d2:::benchmark_data_smithi001_255951_object55489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.602210+0000", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.2478873, 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024058204, 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55490 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.978 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602210+0000", 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602210+0000", 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602211+0000", 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5099999999999999e-07 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602233+0000", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1980999999999998e-05 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602241+0000", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2120000000000003e-06 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.602246+0000", 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1209999999999998e-06 2024-09-14T04:18:38.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.604049+0000", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0018035239999999999 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.604129+0000", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9835999999999999e-05 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.604242+0000", 2024-09-14T04:18:38.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000112253 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.606494+0000", 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002252357 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.613542+0000", 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0070485979999999997 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.982 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626174+0000", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012631552000000001 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626232+0000", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7772000000000002e-05 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626268+0000", 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6347000000000003e-05 2024-09-14T04:18:38.983 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55499 12.e 12:7b3ed6b8:::benchmark_data_smithi001_255951_object55498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.607457+0000", 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.24264017199999999, 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018926423000000001, 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.984 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55499 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607457+0000", 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607457+0000", 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607458+0000", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2699999999999999e-07 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607498+0000", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0119e-05 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607505+0000", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0090000000000001e-06 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607510+0000", 2024-09-14T04:18:38.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2009999999999998e-06 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607553+0000", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2840999999999999e-05 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607700+0000", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014771900000000001 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.607823+0000", 2024-09-14T04:18:38.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012228400000000001 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.609856+0000", 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020331619999999998 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.613597+0000", 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00374144 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.988 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626303+0000", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012706131000000001 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626352+0000", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.846e-05 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.626383+0000", 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1529999999999998e-05 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.989 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:55694 12.3 12:cc9624b0:::benchmark_data_smithi001_255951_object55693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:18:38.727567+0000", 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.12253069699999999, 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.019926208000000001, 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 55694 2024-09-14T04:18:38.990 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727567+0000", 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727567+0000", 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.991 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727571+0000", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0620000000000002e-06 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727603+0000", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2311999999999998e-05 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727619+0000", 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6138e-05 2024-09-14T04:18:38.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.727625+0000", 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4180000000000002e-06 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.729959+0000", 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023335389999999999 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:18:38.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.730085+0000", 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012645500000000001 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.730220+0000", 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013457599999999999 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.732522+0000", 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023019199999999998 2024-09-14T04:18:38.994 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.737708+0000", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051857149999999996 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.747406+0000", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0096982570000000001 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.747452+0000", 2024-09-14T04:18:38.995 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6273000000000002e-05 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:18:38.747493+0000", 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0543000000000001e-05 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:18:38.996 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:18:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:38.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:38.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:39.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:18:39.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:39.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:39.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:39.279 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:39.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:39.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:39.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:39.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:39.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 57077 57061 162.084 106.562 0.00887563 0.00616447 2024-09-14T04:18:39.552 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:39.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:39.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:39.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:39.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:39.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:39.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:39.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:40.077 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:40.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:40.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.245 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:40.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:40.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:18:40.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:40.380 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:40.482 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:40.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:40.532 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 58841 58825 159.83 110.25 0.0086201 0.00625262 2024-09-14T04:18:40.560 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:40 smithi001 ceph-mon[107399]: pgmap v1040: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s; 29329/158865 objects degraded (18.462%) 2024-09-14T04:18:40.560 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:40 smithi001 ceph-mon[112165]: pgmap v1040: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s; 29329/158865 objects degraded (18.462%) 2024-09-14T04:18:40.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:40.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:40.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:40.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:40 smithi012 ceph-mon[101180]: pgmap v1040: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s; 29329/158865 objects degraded (18.462%) 2024-09-14T04:18:40.880 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:40.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:40.906 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:40.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:40.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:41.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:41.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:41.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:41.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:41.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:41.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:41.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:41.458 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-14T04:18:41.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/881884290' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.477 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.478 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/881884290' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.478 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:41.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 60330 60314 157.047 93.0625 0.00564057 0.00636264 2024-09-14T04:18:41.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.620 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:41.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:41.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:41.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/881884290' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:18:41.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:41.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:41.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:42.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:42.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:42.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:42.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:42.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:42.283 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:42.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:42.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:42.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:42.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 62147 62131 155.307 113.562 0.0029799 0.00643452 2024-09-14T04:18:42.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:42.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:42.691 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[107399]: pgmap v1041: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 31563/169119 objects degraded (18.663%) 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[107399]: osdmap e288: 8 total, 7 up, 8 in 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[112165]: pgmap v1041: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 31563/169119 objects degraded (18.663%) 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:18:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:42 smithi001 ceph-mon[112165]: osdmap e288: 8 total, 7 up, 8 in 2024-09-14T04:18:42.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:42.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:42 smithi012 ceph-mon[101180]: pgmap v1041: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s; 31563/169119 objects degraded (18.663%) 2024-09-14T04:18:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:18:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:42 smithi012 ceph-mon[101180]: osdmap e288: 8 total, 7 up, 8 in 2024-09-14T04:18:42.866 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:42.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:42.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:43.007 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:43.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:43.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:43.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:43.258 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:43.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:43.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:43.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:43.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 63970 63954 153.716 113.938 0.00713908 0.00650094 2024-09-14T04:18:43.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:43.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:43.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:43 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 33224/177381 objects degraded (18.730%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:43 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 33224/177381 objects degraded (18.730%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:43.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:43.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:43.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:43 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 33224/177381 objects degraded (18.730%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:43.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:43.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:43.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:44.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:44.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:44.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:44.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:44.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:44.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:44.430 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:44.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:44.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 65764 65748 152.175 112.125 0.00398128 0.00656635 2024-09-14T04:18:44.565 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:44.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:44.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:44.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:44.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:44 smithi012 ceph-mon[101180]: pgmap v1043: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s; 33224/177381 objects degraded (18.730%) 2024-09-14T04:18:44.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:44.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:44 smithi001 ceph-mon[112165]: pgmap v1043: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s; 33224/177381 objects degraded (18.730%) 2024-09-14T04:18:44.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:44 smithi001 ceph-mon[107399]: pgmap v1043: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s; 33224/177381 objects degraded (18.730%) 2024-09-14T04:18:44.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:44.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:45.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:45.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:45.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:45.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:45.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:45.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:45.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:45.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:45.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 67594 67578 150.825 114.375 0.00334295 0.00662568 2024-09-14T04:18:45.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:45.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:45.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:45.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:45.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:45.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:45.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:46.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:46.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:46.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:46.230 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:46.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:46.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:46.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:46.509 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:46.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 69350 69334 149.406 109.75 0.00452244 0.00668898 2024-09-14T04:18:46.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:46.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:46.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:46 smithi001 ceph-mon[107399]: pgmap v1044: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 34289/185475 objects degraded (18.487%) 2024-09-14T04:18:46.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:46 smithi001 ceph-mon[112165]: pgmap v1044: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 34289/185475 objects degraded (18.487%) 2024-09-14T04:18:46.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:46.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:46 smithi012 ceph-mon[101180]: pgmap v1044: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 34289/185475 objects degraded (18.487%) 2024-09-14T04:18:46.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:46.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:46.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:46.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:46] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:18:47.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:47.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:47.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:47.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:47.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:47.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:47.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:47.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:47.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 70844 70828 147.538 93.375 0.0083744 0.00677335 2024-09-14T04:18:47.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:47.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:47.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:47.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:47.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:47.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:47.899 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:47.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:47.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:48.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:48.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:48.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:48.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:48.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:48.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:48.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:48.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:48.531 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:18:48.533 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 72496 72480 146.109 103.25 0.016304 0.00683907 2024-09-14T04:18:48.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:48.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:48.734 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:48.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:48 smithi001 ceph-mon[107399]: pgmap v1045: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 38254/203745 objects degraded (18.775%) 2024-09-14T04:18:48.823 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:48 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:18:48.823 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:48 smithi001 ceph-mon[112165]: pgmap v1045: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 38254/203745 objects degraded (18.775%) 2024-09-14T04:18:48.824 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:48 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:18:48.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:48 smithi012 ceph-mon[101180]: pgmap v1045: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 38254/203745 objects degraded (18.775%) 2024-09-14T04:18:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:48 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:18:48.881 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:48.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:48.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:49.005 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:18:49.005 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:18:49.006 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:18:49.006 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:18:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.030 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:49.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:49.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:49.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:49.237 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2024-09-14T04:18:49.237 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:18:49.237 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T04:18:49.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:18:49.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:49.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:49.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:49.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:49.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 74062 74046 144.6 97.875 0.0102384 0.00691141 2024-09-14T04:18:49.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:49.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:49.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: osdmap e289: 8 total, 7 up, 8 in 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 38254/203745 objects degraded (18.775%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: osdmap e289: 8 total, 7 up, 8 in 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 38254/203745 objects degraded (18.775%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:49.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:49.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3020583669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: osdmap e289: 8 total, 7 up, 8 in 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 38254/203745 objects degraded (18.775%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:49.857 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:49.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:49.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:49.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:49.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:50.095 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:50.128 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:50.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:50.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.266 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:50.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:50.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:50.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:50.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:50.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 75656 75640 143.237 99.625 0.0082314 0.00697686 2024-09-14T04:18:50.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:50.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:50.683 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:50.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.816 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:50.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:50 smithi012 ceph-mon[101180]: pgmap v1047: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 39324/211572 objects degraded (18.587%) 2024-09-14T04:18:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:50.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:50.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:50.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:50.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[107399]: pgmap v1047: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 39324/211572 objects degraded (18.587%) 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[112165]: pgmap v1047: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 39324/211572 objects degraded (18.587%) 2024-09-14T04:18:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:18:51.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:51.082 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:51.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:51.216 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:51.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:51.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:51.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:51.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:51.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:51.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 77181 77165 141.827 95.3125 0.00318029 0.00704559 2024-09-14T04:18:51.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:51.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:51.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:51.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:51.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:18:51.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:51.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:51.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:52.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:52.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:52.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:52.175 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:52.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:52.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:52.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:52.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:52.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 78625 78609 140.354 90.25 0.00543818 0.00712031 2024-09-14T04:18:52.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:52.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:52.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:52.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:52.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:52.824 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:52 smithi012 ceph-mon[101180]: pgmap v1048: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 41293/220452 objects degraded (18.731%) 2024-09-14T04:18:52.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:52.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:52.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:52 smithi001 ceph-mon[107399]: pgmap v1048: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 41293/220452 objects degraded (18.731%) 2024-09-14T04:18:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:52 smithi001 ceph-mon[112165]: pgmap v1048: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 41293/220452 objects degraded (18.731%) 2024-09-14T04:18:53.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:53.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:53.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:53.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:53.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:53.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:53.385 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:53.420 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:53.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:53.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 80089 80073 138.996 91.5 0.0145011 0.00718936 2024-09-14T04:18:53.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:53.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:53.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:53.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:53.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:53.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:53.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:53.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:54.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:54.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:54.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:54.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:54.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:54.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:54.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:54.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:54.504 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:54.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 81416 81400 137.481 82.9375 0.00473195 0.00726873 2024-09-14T04:18:54.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:54.629 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:54.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:54.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:54.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:54.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:54 smithi012 ceph-mon[101180]: pgmap v1049: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 42862/228111 objects degraded (18.790%) 2024-09-14T04:18:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 42862/228111 objects degraded (18.790%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:54.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:54.900 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:54.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:54 smithi001 ceph-mon[107399]: pgmap v1049: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 42862/228111 objects degraded (18.790%) 2024-09-14T04:18:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 42862/228111 objects degraded (18.790%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:54 smithi001 ceph-mon[112165]: pgmap v1049: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 42862/228111 objects degraded (18.790%) 2024-09-14T04:18:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 42862/228111 objects degraded (18.790%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:55.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:55.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:55.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:55.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:55.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:55.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:55.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:55.447 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:55.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:55.534 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 82728 82712 136.021 82 0.00530315 0.00734717 2024-09-14T04:18:55.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:55.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:55.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:55.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:55.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:55 smithi012 ceph-mon[101180]: pgmap v1050: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 43772/235005 objects degraded (18.626%) 2024-09-14T04:18:55.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:55.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:55 smithi001 ceph-mon[107399]: pgmap v1050: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 43772/235005 objects degraded (18.626%) 2024-09-14T04:18:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:55 smithi001 ceph-mon[112165]: pgmap v1050: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 43772/235005 objects degraded (18.626%) 2024-09-14T04:18:55.975 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:55.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:56.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:56.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:56.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:56.222 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:56.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:56.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:56.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:56.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:56.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 83374 83358 133.569 40.375 0.00576005 0.00748216 2024-09-14T04:18:56.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:56.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:56.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:56.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:56.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:56.898 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:56.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:18:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:18:56] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:18:56.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:57.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:57.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:57.169 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:57.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:57.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:57.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:57.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:57.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:57.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:57.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:57.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:18:57.534838+0000 min lat: 0.000960706 max lat: 0.455133 avg lat: 0.00754733 2024-09-14T04:18:57.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:18:57.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 15 84777 84762 132.422 87.75 0.00974105 0.00754733 2024-09-14T04:18:57.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:57.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:57.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:18:57.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:57.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:57.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:57.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:57.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:58.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:58.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:58.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:58.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:58.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:58.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:58.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:58.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:58.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 86202 86186 131.363 89 0.015462 0.00760787 2024-09-14T04:18:58.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:18:58.616 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:58.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:18:58.634 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:18:58.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:18:58.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:58 smithi001 ceph-mon[107399]: pgmap v1051: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 46552/248073 objects degraded (18.765%) 2024-09-14T04:18:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:58 smithi001 ceph-mon[112165]: pgmap v1051: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 46552/248073 objects degraded (18.765%) 2024-09-14T04:18:58.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:18:58.805 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:18:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:58 smithi012 ceph-mon[101180]: pgmap v1051: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 46552/248073 objects degraded (18.765%) 2024-09-14T04:18:58.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:58.943 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:58.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:59.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:59.083 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:59.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:18:59.222 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:18:59.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:18:59.358 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:18:59.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:59.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:59.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 15 87549 87534 130.241 84.25 0.00701612 0.00767336 2024-09-14T04:18:59.578 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:18:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 46552/248073 objects degraded (18.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:59.579 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:18:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 46552/248073 objects degraded (18.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:59.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:18:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.638 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:59.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:18:59.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:18:59.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:18:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:18:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:18:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 46552/248073 objects degraded (18.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:18:59.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:18:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:18:59.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:18:59.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:00.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:00.061 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:00.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:00.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:00.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:00.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:00.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.339 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:00.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:00.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:00.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:00.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 88993 88977 129.309 90.1875 0.0173118 0.00772916 2024-09-14T04:19:00.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:00.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:00 smithi001 ceph-mon[107399]: pgmap v1052: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 47298/253635 objects degraded (18.648%) 2024-09-14T04:19:00.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:00 smithi001 ceph-mon[112165]: pgmap v1052: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 47298/253635 objects degraded (18.648%) 2024-09-14T04:19:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:00.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:00.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:00.756 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:19:00.757 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:19:00.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:00.798 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:19:00.798 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:19:00.798 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:19:00.837 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:19:00.837 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:19:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:00 smithi012 ceph-mon[101180]: pgmap v1052: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 47298/253635 objects degraded (18.648%) 2024-09-14T04:19:00.871 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:19:00.871 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:19:00.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:00.897 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:19:00.898 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:19:00.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:00.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:00.924 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:19:00.925 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:19:00.958 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:19:00.958 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:19:01.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:01.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:01.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:01.195 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:01.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:01.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:01.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.479 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:01.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:01.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 90430 90414 128.412 89.8125 0.00727205 0.00778291 2024-09-14T04:19:01.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:01.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:01.711 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2948272904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:01.711 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2948272904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:01.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:01.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2948272904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:01.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:01.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:01.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:01.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:02.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:02.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:02.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:02.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:02.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:02.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:02.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:02.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 91849 91833 127.529 88.6875 0.00523267 0.00783693 2024-09-14T04:19:02.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:02.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:02.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:02.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:02 smithi001 ceph-mon[107399]: pgmap v1053: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 49063/261603 objects degraded (18.755%) 2024-09-14T04:19:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:02 smithi001 ceph-mon[112165]: pgmap v1053: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 49063/261603 objects degraded (18.755%) 2024-09-14T04:19:02.725 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:02.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:02 smithi012 ceph-mon[101180]: pgmap v1053: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 49063/261603 objects degraded (18.755%) 2024-09-14T04:19:02.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:02.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:02.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:02.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:03.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:03.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:03.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:03.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:19:03.356 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:19:03.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:03.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.408 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:03.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:03.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:03.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 93218 93202 126.616 85.5625 0.0108347 0.00789301 2024-09-14T04:19:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:03.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:03.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:03.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:03.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:03.831 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:03.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:03.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:03.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:03.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:04.016 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:19:04.018 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:19:04.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:04.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:04.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:04.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:04.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:04.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:04.498 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.531 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:04.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:04.535 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 94600 94584 125.76 86.375 0.0168755 0.00794697 2024-09-14T04:19:04.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:04.672 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:04.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:04 smithi001 ceph-mon[112165]: pgmap v1054: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 50361/268119 objects degraded (18.783%) 2024-09-14T04:19:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 50361/268119 objects degraded (18.783%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:04 smithi001 ceph-mon[107399]: pgmap v1054: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 50361/268119 objects degraded (18.783%) 2024-09-14T04:19:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 50361/268119 objects degraded (18.783%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:04.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:04.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:04 smithi012 ceph-mon[101180]: pgmap v1054: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 50361/268119 objects degraded (18.783%) 2024-09-14T04:19:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 50361/268119 objects degraded (18.783%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:04.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:04.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:04.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:05.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:05.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:05.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:05.217 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:05.319 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:05.345 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1054,"stamp":"2024-09-14T04:19:03.334998+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5856040933,"num_objects":89373,"num_object_clones":0,"num_object_copies":268119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50361,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89411,"num_write_kb":5718241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":919595,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91045,"ondisk_log_size":91045,"up":264,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22922912,"kb_used_data":16640032,"kb_used_omap":33,"kb_used_meta":6282782,"kb_avail":727038304,"statfs":{"total":767960285184,"available":744487223296,"internally_reserved":0,"allocated":17039392768,"data_stored":17024708383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34380,"internal_metadata":6433569204},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,0,6,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1041301504,"num_objects":15889,"num_object_clones":0,"num_object_copies":47667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9068,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15889,"num_write_kb":1016896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005890"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":141,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.452548+0000","last_change":"2024-09-14T04:15:33.427476+0000","last_active":"2024-09-14T04:18:41.452548+0000","last_peered":"2024-09-14T04:18:41.452548+0000","last_clean":"2024-09-14T04:06:38.698680+0000","last_became_active":"2024-09-14T04:15:33.427476+0000","last_became_peered":"2024-09-14T04:15:33.427476+0000","last_unstale":"2024-09-14T04:18:41.452548+0000","last_undegraded":"2024-09-14T04:18:41.452548+0000","last_fullsized":"2024-09-14T04:15:33.421753+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:56.824079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":327,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:18:48.545777+0000","last_change":"2024-09-14T04:15:33.426266+0000","last_active":"2024-09-14T04:18:48.545777+0000","last_peered":"2024-09-14T04:18:48.545777+0000","last_clean":"2024-09-14T04:15:30.406073+0000","last_became_active":"2024-09-14T04:15:33.426266+0000","last_became_peered":"2024-09-14T04:15:33.426266+0000","last_unstale":"2024-09-14T04:18:48.545777+0000","last_undegraded":"2024-09-14T04:15:33.422384+0000","last_fullsized":"2024-09-14T04:15:33.422276+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:42.217520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":101,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.462378+0000","last_change":"2024-09-14T04:09:50.219804+0000","last_active":"2024-09-14T04:18:41.462378+0000","last_peered":"2024-09-14T04:18:41.462378+0000","last_clean":"2024-09-14T04:18:41.462378+0000","last_became_active":"2024-09-14T04:09:50.219383+0000","last_became_peered":"2024-09-14T04:09:50.219383+0000","last_unstale":"2024-09-14T04:18:41.462378+0000","last_undegraded":"2024-09-14T04:18:41.462378+0000","last_fullsized":"2024-09-14T04:18:41.462378+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:27:08.790275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":173,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.544859+0000","last_change":"2024-09-14T04:10:35.385940+0000","last_active":"2024-09-14T04:18:48.544859+0000","last_peered":"2024-09-14T04:18:48.544859+0000","last_clean":"2024-09-14T04:18:48.544859+0000","last_became_active":"2024-09-14T04:10:35.385027+0000","last_became_peered":"2024-09-14T04:10:35.385027+0000","last_unstale":"2024-09-14T04:18:48.544859+0000","last_undegraded":"2024-09-14T04:18:48.544859+0000","last_fullsized":"2024-09-14T04:18:48.544859+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:40.926461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":173,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.546530+0000","last_change":"2024-09-14T04:10:35.387447+0000","last_active":"2024-09-14T04:18:48.546530+0000","last_peered":"2024-09-14T04:18:48.546530+0000","last_clean":"2024-09-14T04:18:48.546530+0000","last_became_active":"2024-09-14T04:10:35.386876+0000","last_became_peered":"2024-09-14T04:10:35.386876+0000","last_unstale":"2024-09-14T04:18:48.546530+0000","last_undegraded":"2024-09-14T04:18:48.546530+0000","last_fullsized":"2024-09-14T04:18:48.546530+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:14.318287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.453227+0000","last_change":"2024-09-14T03:42:34.646837+0000","last_active":"2024-09-14T04:18:41.453227+0000","last_peered":"2024-09-14T04:18:41.453227+0000","last_clean":"2024-09-14T04:18:41.453227+0000","last_became_active":"2024-09-14T03:42:34.646438+0000","last_became_peered":"2024-09-14T03:42:34.646438+0000","last_unstale":"2024-09-14T04:18:41.453227+0000","last_undegraded":"2024-09-14T04:18:41.453227+0000","last_fullsized":"2024-09-14T04:18:41.453227+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:56:13.538951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":281,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.541131+0000","last_change":"2024-09-14T04:06:46.345598+0000","last_active":"2024-09-14T04:18:48.541131+0000","last_peered":"2024-09-14T04:18:48.541131+0000","last_clean":"2024-09-14T04:18:48.541131+0000","last_became_active":"2024-09-14T04:06:46.345296+0000","last_became_peered":"2024-09-14T04:06:46.345296+0000","last_unstale":"2024-09-14T04:18:48.541131+0000","last_undegraded":"2024-09-14T04:18:48.541131+0000","last_fullsized":"2024-09-14T04:18:48.541131+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:03.208350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":342,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.455428+0000","last_change":"2024-09-14T04:10:35.395294+0000","last_active":"2024-09-14T04:18:41.455428+0000","last_peered":"2024-09-14T04:18:41.455428+0000","last_clean":"2024-09-14T04:18:41.455428+0000","last_became_active":"2024-09-14T04:10:35.381829+0000","last_became_peered":"2024-09-14T04:10:35.381829+0000","last_unstale":"2024-09-14T04:18:41.455428+0000","last_undegraded":"2024-09-14T04:18:41.455428+0000","last_fullsized":"2024-09-14T04:18:41.455428+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.521297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"289'5592","reported_seq":5604,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.141780+0000","last_change":"2024-09-14T04:18:17.547529+0000","last_active":"2024-09-14T04:19:01.141780+0000","last_peered":"2024-09-14T04:19:01.141780+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.381914+0000","last_became_peered":"2024-09-14T04:18:09.381914+0000","last_unstale":"2024-09-14T04:19:01.141780+0000","last_undegraded":"2024-09-14T04:18:17.541958+0000","last_fullsized":"2024-09-14T04:18:09.378421+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5592,"log_dups_size":0,"ondisk_log_size":5592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:12.639618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366477312,"num_objects":5592,"num_object_clones":0,"num_object_copies":16776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5592,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":357888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":5592}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":303,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:46.792456+0000","last_change":"2024-09-14T04:10:35.402040+0000","last_active":"2024-09-14T04:18:46.792456+0000","last_peered":"2024-09-14T04:18:46.792456+0000","last_clean":"2024-09-14T04:18:46.792456+0000","last_became_active":"2024-09-14T04:10:35.388982+0000","last_became_peered":"2024-09-14T04:10:35.388982+0000","last_unstale":"2024-09-14T04:18:46.792456+0000","last_undegraded":"2024-09-14T04:18:46.792456+0000","last_fullsized":"2024-09-14T04:18:46.792456+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:59:53.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":111,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.461787+0000","last_change":"2024-09-14T04:14:43.297521+0000","last_active":"2024-09-14T04:18:41.461787+0000","last_peered":"2024-09-14T04:18:41.461787+0000","last_clean":"2024-09-14T04:18:41.461787+0000","last_became_active":"2024-09-14T04:14:43.297023+0000","last_became_peered":"2024-09-14T04:14:43.297023+0000","last_unstale":"2024-09-14T04:18:41.461787+0000","last_undegraded":"2024-09-14T04:18:41.461787+0000","last_fullsized":"2024-09-14T04:18:41.461787+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:56.503762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"289'5598","reported_seq":5621,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:19:01.211447+0000","last_change":"2024-09-14T04:18:09.383146+0000","last_active":"2024-09-14T04:19:01.211447+0000","last_peered":"2024-09-14T04:19:01.211447+0000","last_clean":"2024-09-14T04:19:01.211447+0000","last_became_active":"2024-09-14T04:18:09.381740+0000","last_became_peered":"2024-09-14T04:18:09.381740+0000","last_unstale":"2024-09-14T04:19:01.211447+0000","last_undegraded":"2024-09-14T04:19:01.211447+0000","last_fullsized":"2024-09-14T04:19:01.211447+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:11:51.337727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366870528,"num_objects":5598,"num_object_clones":0,"num_object_copies":16794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":358272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":302,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.686755+0000","last_change":"2024-09-14T04:11:13.253112+0000","last_active":"2024-09-14T04:18:41.686755+0000","last_peered":"2024-09-14T04:18:41.686755+0000","last_clean":"2024-09-14T04:18:41.686755+0000","last_became_active":"2024-09-14T04:11:13.250777+0000","last_became_peered":"2024-09-14T04:11:13.250777+0000","last_unstale":"2024-09-14T04:18:41.686755+0000","last_undegraded":"2024-09-14T04:18:41.686755+0000","last_fullsized":"2024-09-14T04:18:41.686755+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:36:29.487356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":293,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.459274+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:18:41.459274+0000","last_peered":"2024-09-14T04:18:41.459274+0000","last_clean":"2024-09-14T04:18:41.459274+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:18:41.459274+0000","last_undegraded":"2024-09-14T04:18:41.459274+0000","last_fullsized":"2024-09-14T04:18:41.459274+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"289'5638","reported_seq":5653,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:19:01.146184+0000","last_change":"2024-09-14T04:18:09.383448+0000","last_active":"2024-09-14T04:19:01.146184+0000","last_peered":"2024-09-14T04:19:01.146184+0000","last_clean":"2024-09-14T04:19:01.146184+0000","last_became_active":"2024-09-14T04:18:09.383130+0000","last_became_peered":"2024-09-14T04:18:09.383130+0000","last_unstale":"2024-09-14T04:19:01.146184+0000","last_undegraded":"2024-09-14T04:19:01.146184+0000","last_fullsized":"2024-09-14T04:19:01.146184+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:10.231674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369491968,"num_objects":5638,"num_object_clones":0,"num_object_copies":16914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5638,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":273,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.686737+0000","last_change":"2024-09-14T04:09:50.229004+0000","last_active":"2024-09-14T04:18:41.686737+0000","last_peered":"2024-09-14T04:18:41.686737+0000","last_clean":"2024-09-14T04:18:41.686737+0000","last_became_active":"2024-09-14T04:09:50.223403+0000","last_became_peered":"2024-09-14T04:09:50.223403+0000","last_unstale":"2024-09-14T04:18:41.686737+0000","last_undegraded":"2024-09-14T04:18:41.686737+0000","last_fullsized":"2024-09-14T04:18:41.686737+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:11:46.966820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.543350+0000","last_change":"2024-09-14T04:14:43.299260+0000","last_active":"2024-09-14T04:18:48.543350+0000","last_peered":"2024-09-14T04:18:48.543350+0000","last_clean":"2024-09-14T04:18:48.543350+0000","last_became_active":"2024-09-14T04:14:43.298810+0000","last_became_peered":"2024-09-14T04:14:43.298810+0000","last_unstale":"2024-09-14T04:18:48.543350+0000","last_undegraded":"2024-09-14T04:18:48.543350+0000","last_fullsized":"2024-09-14T04:18:48.543350+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:30.650285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.3","version":"289'5348","reported_seq":5371,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:59.064644+0000","last_change":"2024-09-14T04:18:09.385298+0000","last_active":"2024-09-14T04:18:59.064644+0000","last_peered":"2024-09-14T04:18:59.064644+0000","last_clean":"2024-09-14T04:18:59.064644+0000","last_became_active":"2024-09-14T04:18:09.384932+0000","last_became_peered":"2024-09-14T04:18:09.384932+0000","last_unstale":"2024-09-14T04:18:59.064644+0000","last_undegraded":"2024-09-14T04:18:59.064644+0000","last_fullsized":"2024-09-14T04:18:59.064644+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:16.345489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350486528,"num_objects":5348,"num_object_clones":0,"num_object_copies":16044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5348,"num_write_kb":342272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":279,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.462408+0000","last_change":"2024-09-14T04:06:46.338623+0000","last_active":"2024-09-14T04:18:41.462408+0000","last_peered":"2024-09-14T04:18:41.462408+0000","last_clean":"2024-09-14T04:18:41.462408+0000","last_became_active":"2024-09-14T04:06:46.338282+0000","last_became_peered":"2024-09-14T04:06:46.338282+0000","last_unstale":"2024-09-14T04:18:41.462408+0000","last_undegraded":"2024-09-14T04:18:41.462408+0000","last_fullsized":"2024-09-14T04:18:41.462408+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:50.732900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":173,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.546065+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:18:48.546065+0000","last_peered":"2024-09-14T04:18:48.546065+0000","last_clean":"2024-09-14T04:18:48.546065+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:18:48.546065+0000","last_undegraded":"2024-09-14T04:18:48.546065+0000","last_fullsized":"2024-09-14T04:18:48.546065+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"289'5793","reported_seq":5808,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:19:01.899065+0000","last_change":"2024-09-14T04:18:09.384592+0000","last_active":"2024-09-14T04:19:01.899065+0000","last_peered":"2024-09-14T04:19:01.899065+0000","last_clean":"2024-09-14T04:19:01.899065+0000","last_became_active":"2024-09-14T04:18:09.384277+0000","last_became_peered":"2024-09-14T04:18:09.384277+0000","last_unstale":"2024-09-14T04:19:01.899065+0000","last_undegraded":"2024-09-14T04:19:01.899065+0000","last_fullsized":"2024-09-14T04:19:01.899065+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:03.957577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379650048,"num_objects":5793,"num_object_clones":0,"num_object_copies":17379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5793,"num_write_kb":370752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":650,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.686598+0000","last_change":"2024-09-14T04:15:33.426857+0000","last_active":"2024-09-14T04:18:41.686598+0000","last_peered":"2024-09-14T04:18:41.686598+0000","last_clean":"2024-09-14T04:15:30.401586+0000","last_became_active":"2024-09-14T04:15:33.426857+0000","last_became_peered":"2024-09-14T04:15:33.426857+0000","last_unstale":"2024-09-14T04:18:41.686598+0000","last_undegraded":"2024-09-14T04:18:41.686598+0000","last_fullsized":"2024-09-14T04:15:33.421178+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.948666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.948666+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:21.288362+0000","scrub_duration":0.001948782,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":177,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.545353+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:18:48.545353+0000","last_peered":"2024-09-14T04:18:48.545353+0000","last_clean":"2024-09-14T04:18:48.545353+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:18:48.545353+0000","last_undegraded":"2024-09-14T04:18:48.545353+0000","last_fullsized":"2024-09-14T04:18:48.545353+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":287,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.459186+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:18:41.459186+0000","last_peered":"2024-09-14T04:18:41.459186+0000","last_clean":"2024-09-14T04:18:41.459186+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:18:41.459186+0000","last_undegraded":"2024-09-14T04:18:41.459186+0000","last_fullsized":"2024-09-14T04:18:41.459186+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"289'5671","reported_seq":5683,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.875492+0000","last_change":"2024-09-14T04:18:17.538444+0000","last_active":"2024-09-14T04:19:01.875492+0000","last_peered":"2024-09-14T04:19:01.875492+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.381469+0000","last_became_peered":"2024-09-14T04:18:09.381469+0000","last_unstale":"2024-09-14T04:19:01.875492+0000","last_undegraded":"2024-09-14T04:18:17.535076+0000","last_fullsized":"2024-09-14T04:18:09.378212+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:34:04.513340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371654656,"num_objects":5671,"num_object_clones":0,"num_object_copies":17013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5671,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5671,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":5671}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":519,"reported_epoch":288,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:18:41.461653+0000","last_change":"2024-09-14T04:15:33.422606+0000","last_active":"2024-09-14T04:18:41.461653+0000","last_peered":"2024-09-14T04:18:41.461653+0000","last_clean":"2024-09-14T04:15:30.401002+0000","last_became_active":"2024-09-14T04:15:33.422606+0000","last_became_peered":"2024-09-14T04:15:33.422606+0000","last_unstale":"2024-09-14T04:18:41.461653+0000","last_undegraded":"2024-09-14T04:15:33.418929+0000","last_fullsized":"2024-09-14T04:15:33.418761+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:58.946948+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":303,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.455792+0000","last_change":"2024-09-14T04:16:40.484372+0000","last_active":"2024-09-14T04:18:41.455792+0000","last_peered":"2024-09-14T04:18:41.455792+0000","last_clean":"2024-09-14T04:18:41.455792+0000","last_became_active":"2024-09-14T04:16:40.483930+0000","last_became_peered":"2024-09-14T04:16:40.483930+0000","last_unstale":"2024-09-14T04:18:41.455792+0000","last_undegraded":"2024-09-14T04:18:41.455792+0000","last_fullsized":"2024-09-14T04:18:41.455792+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:49:55.941784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":88,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.460914+0000","last_change":"2024-09-14T04:15:33.434693+0000","last_active":"2024-09-14T04:18:41.460914+0000","last_peered":"2024-09-14T04:18:41.460914+0000","last_clean":"2024-09-14T04:15:30.424099+0000","last_became_active":"2024-09-14T04:15:33.434693+0000","last_became_peered":"2024-09-14T04:15:33.434693+0000","last_unstale":"2024-09-14T04:18:41.460914+0000","last_undegraded":"2024-09-14T04:18:41.460914+0000","last_fullsized":"2024-09-14T04:15:33.422888+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:03:32.435182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"289'5552","reported_seq":5575,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:59.068484+0000","last_change":"2024-09-14T04:18:09.381893+0000","last_active":"2024-09-14T04:18:59.068484+0000","last_peered":"2024-09-14T04:18:59.068484+0000","last_clean":"2024-09-14T04:18:59.068484+0000","last_became_active":"2024-09-14T04:18:09.381576+0000","last_became_peered":"2024-09-14T04:18:09.381576+0000","last_unstale":"2024-09-14T04:18:59.068484+0000","last_undegraded":"2024-09-14T04:18:59.068484+0000","last_fullsized":"2024-09-14T04:18:59.068484+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:34.999471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363855872,"num_objects":5552,"num_object_clones":0,"num_object_copies":16656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5552,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.459217+0000","last_change":"2024-09-14T04:15:33.425078+0000","last_active":"2024-09-14T04:18:41.459217+0000","last_peered":"2024-09-14T04:18:41.459217+0000","last_clean":"2024-09-14T04:15:30.423042+0000","last_became_active":"2024-09-14T04:15:33.425078+0000","last_became_peered":"2024-09-14T04:15:33.425078+0000","last_unstale":"2024-09-14T04:18:41.459217+0000","last_undegraded":"2024-09-14T04:18:41.459217+0000","last_fullsized":"2024-09-14T04:15:33.421558+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:13:19.510962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":278,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.460960+0000","last_change":"2024-09-14T04:16:40.485632+0000","last_active":"2024-09-14T04:18:41.460960+0000","last_peered":"2024-09-14T04:18:41.460960+0000","last_clean":"2024-09-14T04:18:41.460960+0000","last_became_active":"2024-09-14T04:16:40.485221+0000","last_became_peered":"2024-09-14T04:16:40.485221+0000","last_unstale":"2024-09-14T04:18:41.460960+0000","last_undegraded":"2024-09-14T04:18:41.460960+0000","last_fullsized":"2024-09-14T04:18:41.460960+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:36.620868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"289'5715","reported_seq":5727,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.780583+0000","last_change":"2024-09-14T04:18:17.533326+0000","last_active":"2024-09-14T04:19:01.780583+0000","last_peered":"2024-09-14T04:19:01.780583+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.383061+0000","last_became_peered":"2024-09-14T04:18:09.383061+0000","last_unstale":"2024-09-14T04:19:01.780583+0000","last_undegraded":"2024-09-14T04:18:17.532873+0000","last_fullsized":"2024-09-14T04:18:09.379407+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:00.089260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374538240,"num_objects":5715,"num_object_clones":0,"num_object_copies":17145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5715,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":5715}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":502,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.462025+0000","last_change":"2024-09-14T04:15:33.424214+0000","last_active":"2024-09-14T04:18:41.462025+0000","last_peered":"2024-09-14T04:18:41.462025+0000","last_clean":"2024-09-14T04:15:30.401178+0000","last_became_active":"2024-09-14T04:15:33.424214+0000","last_became_peered":"2024-09-14T04:15:33.424214+0000","last_unstale":"2024-09-14T04:18:41.462025+0000","last_undegraded":"2024-09-14T04:18:41.462025+0000","last_fullsized":"2024-09-14T04:15:33.418859+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:41.327975+0000","scrub_duration":0.002868467,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":281,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.541100+0000","last_change":"2024-09-14T04:06:46.335198+0000","last_active":"2024-09-14T04:18:48.541100+0000","last_peered":"2024-09-14T04:18:48.541100+0000","last_clean":"2024-09-14T04:18:48.541100+0000","last_became_active":"2024-09-14T04:06:46.334848+0000","last_became_peered":"2024-09-14T04:06:46.334848+0000","last_unstale":"2024-09-14T04:18:48.541100+0000","last_undegraded":"2024-09-14T04:18:48.541100+0000","last_fullsized":"2024-09-14T04:18:48.541100+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:23.624863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":417,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.544423+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:18:48.544423+0000","last_peered":"2024-09-14T04:18:48.544423+0000","last_clean":"2024-09-14T04:18:48.544423+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:18:48.544423+0000","last_undegraded":"2024-09-14T04:18:48.544423+0000","last_fullsized":"2024-09-14T04:18:48.544423+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"289'5799","reported_seq":5811,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.769487+0000","last_change":"2024-09-14T04:18:17.533133+0000","last_active":"2024-09-14T04:19:01.769487+0000","last_peered":"2024-09-14T04:19:01.769487+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.381255+0000","last_became_peered":"2024-09-14T04:18:09.381255+0000","last_unstale":"2024-09-14T04:19:01.769487+0000","last_undegraded":"2024-09-14T04:18:17.532343+0000","last_fullsized":"2024-09-14T04:18:09.377763+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:38:38.011458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380043264,"num_objects":5799,"num_object_clones":0,"num_object_copies":17397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5799,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":371136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":5799}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":662,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.539764+0000","last_change":"2024-09-14T04:10:38.470243+0000","last_active":"2024-09-14T04:18:48.539764+0000","last_peered":"2024-09-14T04:18:48.539764+0000","last_clean":"2024-09-14T04:18:48.539764+0000","last_became_active":"2024-09-14T04:10:38.468810+0000","last_became_peered":"2024-09-14T04:10:38.468810+0000","last_unstale":"2024-09-14T04:18:48.539764+0000","last_undegraded":"2024-09-14T04:18:48.539764+0000","last_fullsized":"2024-09-14T04:18:48.539764+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:16.131738+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:16.131738+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:47.857507+0000","scrub_duration":0.001447088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":282,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:18:48.532818+0000","last_change":"2024-09-14T04:15:33.428752+0000","last_active":"2024-09-14T04:18:48.532818+0000","last_peered":"2024-09-14T04:18:48.532818+0000","last_clean":"2024-09-14T04:15:30.401686+0000","last_became_active":"2024-09-14T04:15:33.428752+0000","last_became_peered":"2024-09-14T04:15:33.428752+0000","last_unstale":"2024-09-14T04:18:48.532818+0000","last_undegraded":"2024-09-14T04:15:33.419396+0000","last_fullsized":"2024-09-14T04:15:33.419276+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:34:46.004410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034803200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":223,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:47.299240+0000","last_change":"2024-09-14T04:07:20.035365+0000","last_active":"2024-09-14T04:18:47.299240+0000","last_peered":"2024-09-14T04:18:47.299240+0000","last_clean":"2024-09-14T04:18:47.299240+0000","last_became_active":"2024-09-14T04:07:20.035026+0000","last_became_peered":"2024-09-14T04:07:20.035026+0000","last_unstale":"2024-09-14T04:18:47.299240+0000","last_undegraded":"2024-09-14T04:18:47.299240+0000","last_fullsized":"2024-09-14T04:18:47.299240+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:06:33.856620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.541043+0000","last_change":"2024-09-14T04:14:43.297636+0000","last_active":"2024-09-14T04:18:48.541043+0000","last_peered":"2024-09-14T04:18:48.541043+0000","last_clean":"2024-09-14T04:18:48.541043+0000","last_became_active":"2024-09-14T04:14:43.297227+0000","last_became_peered":"2024-09-14T04:14:43.297227+0000","last_unstale":"2024-09-14T04:18:48.541043+0000","last_undegraded":"2024-09-14T04:18:48.541043+0000","last_fullsized":"2024-09-14T04:18:48.541043+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:46.850117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"289'5416","reported_seq":5439,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:59.085104+0000","last_change":"2024-09-14T04:18:09.383227+0000","last_active":"2024-09-14T04:18:59.085104+0000","last_peered":"2024-09-14T04:18:59.085104+0000","last_clean":"2024-09-14T04:18:59.085104+0000","last_became_active":"2024-09-14T04:18:09.382933+0000","last_became_peered":"2024-09-14T04:18:09.382933+0000","last_unstale":"2024-09-14T04:18:59.085104+0000","last_undegraded":"2024-09-14T04:18:59.085104+0000","last_fullsized":"2024-09-14T04:18:59.085104+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:21:02.877042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354942976,"num_objects":5416,"num_object_clones":0,"num_object_copies":16248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5416,"num_write_kb":346624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":611,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.543656+0000","last_change":"2024-09-14T04:14:46.362726+0000","last_active":"2024-09-14T04:18:48.543656+0000","last_peered":"2024-09-14T04:18:48.543656+0000","last_clean":"2024-09-14T04:18:48.543656+0000","last_became_active":"2024-09-14T04:14:46.362090+0000","last_became_peered":"2024-09-14T04:14:46.362090+0000","last_unstale":"2024-09-14T04:18:48.543656+0000","last_undegraded":"2024-09-14T04:18:48.543656+0000","last_fullsized":"2024-09-14T04:18:48.543656+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:32.190656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:32.190656+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:06.580218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":305,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.453070+0000","last_change":"2024-09-14T04:08:15.319308+0000","last_active":"2024-09-14T04:18:41.453070+0000","last_peered":"2024-09-14T04:18:41.453070+0000","last_clean":"2024-09-14T04:18:41.453070+0000","last_became_active":"2024-09-14T04:08:15.318611+0000","last_became_peered":"2024-09-14T04:08:15.318611+0000","last_unstale":"2024-09-14T04:18:41.453070+0000","last_undegraded":"2024-09-14T04:18:41.453070+0000","last_fullsized":"2024-09-14T04:18:41.453070+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:50.476576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":397,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.542561+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:18:48.542561+0000","last_peered":"2024-09-14T04:18:48.542561+0000","last_clean":"2024-09-14T04:18:48.542561+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:18:48.542561+0000","last_undegraded":"2024-09-14T04:18:48.542561+0000","last_fullsized":"2024-09-14T04:18:48.542561+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.546608+0000","last_change":"2024-09-14T04:15:33.431021+0000","last_active":"2024-09-14T04:18:48.546608+0000","last_peered":"2024-09-14T04:18:48.546608+0000","last_clean":"2024-09-14T04:15:30.410919+0000","last_became_active":"2024-09-14T04:15:33.431021+0000","last_became_peered":"2024-09-14T04:15:33.431021+0000","last_unstale":"2024-09-14T04:18:48.546608+0000","last_undegraded":"2024-09-14T04:18:48.546608+0000","last_fullsized":"2024-09-14T04:15:33.420997+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:31.739487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"289'5645","reported_seq":5660,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:19:02.275995+0000","last_change":"2024-09-14T04:18:09.383026+0000","last_active":"2024-09-14T04:19:02.275995+0000","last_peered":"2024-09-14T04:19:02.275995+0000","last_clean":"2024-09-14T04:19:02.275995+0000","last_became_active":"2024-09-14T04:18:09.382714+0000","last_became_peered":"2024-09-14T04:18:09.382714+0000","last_unstale":"2024-09-14T04:19:02.275995+0000","last_undegraded":"2024-09-14T04:19:02.275995+0000","last_fullsized":"2024-09-14T04:19:02.275995+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:00:59.971966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369950720,"num_objects":5645,"num_object_clones":0,"num_object_copies":16935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":524,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.452330+0000","last_change":"2024-09-14T04:11:16.287644+0000","last_active":"2024-09-14T04:18:41.452330+0000","last_peered":"2024-09-14T04:18:41.452330+0000","last_clean":"2024-09-14T04:18:41.452330+0000","last_became_active":"2024-09-14T04:11:16.287269+0000","last_became_peered":"2024-09-14T04:11:16.287269+0000","last_unstale":"2024-09-14T04:18:41.452330+0000","last_undegraded":"2024-09-14T04:18:41.452330+0000","last_fullsized":"2024-09-14T04:18:41.452330+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:33.344695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:33.344695+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:35.269383+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":284,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.535657+0000","last_change":"2024-09-14T04:15:33.427937+0000","last_active":"2024-09-14T04:18:48.535657+0000","last_peered":"2024-09-14T04:18:48.535657+0000","last_clean":"2024-09-14T04:08:13.357283+0000","last_became_active":"2024-09-14T04:15:33.427937+0000","last_became_peered":"2024-09-14T04:15:33.427937+0000","last_unstale":"2024-09-14T04:18:48.535657+0000","last_undegraded":"2024-09-14T04:18:48.535657+0000","last_fullsized":"2024-09-14T04:15:33.420422+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:11:32.758147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":414,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.543430+0000","last_change":"2024-09-14T04:15:33.430324+0000","last_active":"2024-09-14T04:18:48.543430+0000","last_peered":"2024-09-14T04:18:48.543430+0000","last_clean":"2024-09-14T04:15:30.423647+0000","last_became_active":"2024-09-14T04:15:33.430324+0000","last_became_peered":"2024-09-14T04:15:33.430324+0000","last_unstale":"2024-09-14T04:18:48.543430+0000","last_undegraded":"2024-09-14T04:18:48.543430+0000","last_fullsized":"2024-09-14T04:15:33.425069+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:59.244856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":334,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.533201+0000","last_change":"2024-09-14T04:16:40.485268+0000","last_active":"2024-09-14T04:18:48.533201+0000","last_peered":"2024-09-14T04:18:48.533201+0000","last_clean":"2024-09-14T04:18:48.533201+0000","last_became_active":"2024-09-14T04:16:40.484892+0000","last_became_peered":"2024-09-14T04:16:40.484892+0000","last_unstale":"2024-09-14T04:18:48.533201+0000","last_undegraded":"2024-09-14T04:18:48.533201+0000","last_fullsized":"2024-09-14T04:18:48.533201+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:39.594662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"289'5441","reported_seq":5461,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:18:59.089804+0000","last_change":"2024-09-14T04:18:17.537562+0000","last_active":"2024-09-14T04:18:59.089804+0000","last_peered":"2024-09-14T04:18:59.089804+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.380281+0000","last_became_peered":"2024-09-14T04:18:09.380281+0000","last_unstale":"2024-09-14T04:18:59.089804+0000","last_undegraded":"2024-09-14T04:18:17.537195+0000","last_fullsized":"2024-09-14T04:18:09.376468+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:00.893036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356581376,"num_objects":5441,"num_object_clones":0,"num_object_copies":16323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5441,"num_write_kb":348224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":5441}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":537,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.532503+0000","last_change":"2024-09-14T04:15:33.423459+0000","last_active":"2024-09-14T04:18:48.532503+0000","last_peered":"2024-09-14T04:18:48.532503+0000","last_clean":"2024-09-14T04:15:30.401004+0000","last_became_active":"2024-09-14T04:15:33.423459+0000","last_became_peered":"2024-09-14T04:15:33.423459+0000","last_unstale":"2024-09-14T04:18:48.532503+0000","last_undegraded":"2024-09-14T04:18:48.532503+0000","last_fullsized":"2024-09-14T04:15:33.418755+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.190081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.190081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:05:39.764534+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":290,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.532820+0000","last_change":"2024-09-14T04:14:43.305943+0000","last_active":"2024-09-14T04:18:48.532820+0000","last_peered":"2024-09-14T04:18:48.532820+0000","last_clean":"2024-09-14T04:18:48.532820+0000","last_became_active":"2024-09-14T04:14:43.299556+0000","last_became_peered":"2024-09-14T04:14:43.299556+0000","last_unstale":"2024-09-14T04:18:48.532820+0000","last_undegraded":"2024-09-14T04:18:48.532820+0000","last_fullsized":"2024-09-14T04:18:48.532820+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:00.271277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":295,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.461218+0000","last_change":"2024-09-14T04:10:35.382095+0000","last_active":"2024-09-14T04:18:41.461218+0000","last_peered":"2024-09-14T04:18:41.461218+0000","last_clean":"2024-09-14T04:18:41.461218+0000","last_became_active":"2024-09-14T04:10:35.381732+0000","last_became_peered":"2024-09-14T04:10:35.381732+0000","last_unstale":"2024-09-14T04:18:41.461218+0000","last_undegraded":"2024-09-14T04:18:41.461218+0000","last_fullsized":"2024-09-14T04:18:41.461218+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:00.527535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":297,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:47.299278+0000","last_change":"2024-09-14T04:15:33.425095+0000","last_active":"2024-09-14T04:18:47.299278+0000","last_peered":"2024-09-14T04:18:47.299278+0000","last_clean":"2024-09-14T04:15:30.401324+0000","last_became_active":"2024-09-14T04:15:33.425095+0000","last_became_peered":"2024-09-14T04:15:33.425095+0000","last_unstale":"2024-09-14T04:18:47.299278+0000","last_undegraded":"2024-09-14T04:18:47.299278+0000","last_fullsized":"2024-09-14T04:15:33.418961+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:48:39.065775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.a","version":"289'5515","reported_seq":5535,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:18:59.086195+0000","last_change":"2024-09-14T04:18:17.534145+0000","last_active":"2024-09-14T04:18:59.086195+0000","last_peered":"2024-09-14T04:18:59.086195+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.381196+0000","last_became_peered":"2024-09-14T04:18:09.381196+0000","last_unstale":"2024-09-14T04:18:59.086195+0000","last_undegraded":"2024-09-14T04:18:17.533212+0000","last_fullsized":"2024-09-14T04:18:09.375809+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:46:33.368148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361431040,"num_objects":5515,"num_object_clones":0,"num_object_copies":16545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":352960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":5515}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":461,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.540988+0000","last_change":"2024-09-14T04:14:47.143162+0000","last_active":"2024-09-14T04:18:48.540988+0000","last_peered":"2024-09-14T04:18:48.540988+0000","last_clean":"2024-09-14T04:18:48.540988+0000","last_became_active":"2024-09-14T04:14:46.351157+0000","last_became_peered":"2024-09-14T04:14:46.351157+0000","last_unstale":"2024-09-14T04:18:48.540988+0000","last_undegraded":"2024-09-14T04:18:48.540988+0000","last_fullsized":"2024-09-14T04:18:48.540988+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:14:47.143091+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:14:47.143091+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:53:18.179972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.462632+0000","last_change":"2024-09-14T04:15:33.425722+0000","last_active":"2024-09-14T04:18:41.462632+0000","last_peered":"2024-09-14T04:18:41.462632+0000","last_clean":"2024-09-14T04:15:30.402130+0000","last_became_active":"2024-09-14T04:15:33.425722+0000","last_became_peered":"2024-09-14T04:15:33.425722+0000","last_unstale":"2024-09-14T04:18:41.462632+0000","last_undegraded":"2024-09-14T04:18:41.462632+0000","last_fullsized":"2024-09-14T04:15:33.419799+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:08.888409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":472,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.545448+0000","last_change":"2024-09-14T04:08:15.323401+0000","last_active":"2024-09-14T04:18:48.545448+0000","last_peered":"2024-09-14T04:18:48.545448+0000","last_clean":"2024-09-14T04:18:48.545448+0000","last_became_active":"2024-09-14T04:08:15.322695+0000","last_became_peered":"2024-09-14T04:08:15.322695+0000","last_unstale":"2024-09-14T04:18:48.545448+0000","last_undegraded":"2024-09-14T04:18:48.545448+0000","last_fullsized":"2024-09-14T04:18:48.545448+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:13.704014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":306,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:47.299215+0000","last_change":"2024-09-14T04:16:40.483258+0000","last_active":"2024-09-14T04:18:47.299215+0000","last_peered":"2024-09-14T04:18:47.299215+0000","last_clean":"2024-09-14T04:18:47.299215+0000","last_became_active":"2024-09-14T04:16:40.482985+0000","last_became_peered":"2024-09-14T04:16:40.482985+0000","last_unstale":"2024-09-14T04:18:47.299215+0000","last_undegraded":"2024-09-14T04:18:47.299215+0000","last_fullsized":"2024-09-14T04:18:47.299215+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:31.437686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.b","version":"289'5616","reported_seq":5628,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.156697+0000","last_change":"2024-09-14T04:18:17.544234+0000","last_active":"2024-09-14T04:19:01.156697+0000","last_peered":"2024-09-14T04:19:01.156697+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.382650+0000","last_became_peered":"2024-09-14T04:18:09.382650+0000","last_unstale":"2024-09-14T04:19:01.156697+0000","last_undegraded":"2024-09-14T04:18:17.537019+0000","last_fullsized":"2024-09-14T04:18:09.378419+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:24.524404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":5616}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":538,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.456139+0000","last_change":"2024-09-14T04:10:38.402965+0000","last_active":"2024-09-14T04:18:41.456139+0000","last_peered":"2024-09-14T04:18:41.456139+0000","last_clean":"2024-09-14T04:18:41.456139+0000","last_became_active":"2024-09-14T04:10:38.402547+0000","last_became_peered":"2024-09-14T04:10:38.402547+0000","last_unstale":"2024-09-14T04:18:41.456139+0000","last_undegraded":"2024-09-14T04:18:41.456139+0000","last_fullsized":"2024-09-14T04:18:41.456139+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:34.247210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:34.247210+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:00.065942+0000","scrub_duration":0.001540109,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":407,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.545283+0000","last_change":"2024-09-14T04:15:33.434479+0000","last_active":"2024-09-14T04:18:48.545283+0000","last_peered":"2024-09-14T04:18:48.545283+0000","last_clean":"2024-09-14T04:15:30.423764+0000","last_became_active":"2024-09-14T04:15:33.434479+0000","last_became_peered":"2024-09-14T04:15:33.434479+0000","last_unstale":"2024-09-14T04:18:48.545283+0000","last_undegraded":"2024-09-14T04:18:48.545283+0000","last_fullsized":"2024-09-14T04:15:33.423285+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:25.382562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":308,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.539208+0000","last_change":"2024-09-14T04:10:35.389814+0000","last_active":"2024-09-14T04:18:48.539208+0000","last_peered":"2024-09-14T04:18:48.539208+0000","last_clean":"2024-09-14T04:18:48.539208+0000","last_became_active":"2024-09-14T04:10:35.389279+0000","last_became_peered":"2024-09-14T04:10:35.389279+0000","last_unstale":"2024-09-14T04:18:48.539208+0000","last_undegraded":"2024-09-14T04:18:48.539208+0000","last_fullsized":"2024-09-14T04:18:48.539208+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:52.448287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":174,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.545725+0000","last_change":"2024-09-14T04:10:35.387726+0000","last_active":"2024-09-14T04:18:48.545725+0000","last_peered":"2024-09-14T04:18:48.545725+0000","last_clean":"2024-09-14T04:18:48.545725+0000","last_became_active":"2024-09-14T04:10:35.386923+0000","last_became_peered":"2024-09-14T04:10:35.386923+0000","last_unstale":"2024-09-14T04:18:48.545725+0000","last_undegraded":"2024-09-14T04:18:48.545725+0000","last_fullsized":"2024-09-14T04:18:48.545725+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:22.159734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"289'5482","reported_seq":5502,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.219121+0000","last_change":"2024-09-14T04:18:17.534649+0000","last_active":"2024-09-14T04:19:01.219121+0000","last_peered":"2024-09-14T04:19:01.219121+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.382023+0000","last_became_peered":"2024-09-14T04:18:09.382023+0000","last_unstale":"2024-09-14T04:19:01.219121+0000","last_undegraded":"2024-09-14T04:18:17.534489+0000","last_fullsized":"2024-09-14T04:18:09.378150+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5482,"log_dups_size":0,"ondisk_log_size":5482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:21:15.184766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359268352,"num_objects":5482,"num_object_clones":0,"num_object_copies":16446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5482,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5482,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":5482}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":537,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.541137+0000","last_change":"2024-09-14T04:16:43.524000+0000","last_active":"2024-09-14T04:18:48.541137+0000","last_peered":"2024-09-14T04:18:48.541137+0000","last_clean":"2024-09-14T04:18:48.541137+0000","last_became_active":"2024-09-14T04:16:43.523514+0000","last_became_peered":"2024-09-14T04:16:43.523514+0000","last_unstale":"2024-09-14T04:18:48.541137+0000","last_undegraded":"2024-09-14T04:18:48.541137+0000","last_fullsized":"2024-09-14T04:18:48.541137+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:33.947631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:33.947631+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:26.425372+0000","scrub_duration":0.0013916040000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":225,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.456484+0000","last_change":"2024-09-14T04:10:35.397705+0000","last_active":"2024-09-14T04:18:41.456484+0000","last_peered":"2024-09-14T04:18:41.456484+0000","last_clean":"2024-09-14T04:18:41.456484+0000","last_became_active":"2024-09-14T04:10:35.388760+0000","last_became_peered":"2024-09-14T04:10:35.388760+0000","last_unstale":"2024-09-14T04:18:41.456484+0000","last_undegraded":"2024-09-14T04:18:41.456484+0000","last_fullsized":"2024-09-14T04:18:41.456484+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:27.660621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":403,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.544170+0000","last_change":"2024-09-14T03:57:55.657590+0000","last_active":"2024-09-14T04:18:48.544170+0000","last_peered":"2024-09-14T04:18:48.544170+0000","last_clean":"2024-09-14T04:18:48.544170+0000","last_became_active":"2024-09-14T03:57:55.656321+0000","last_became_peered":"2024-09-14T03:57:55.656321+0000","last_unstale":"2024-09-14T04:18:48.544170+0000","last_undegraded":"2024-09-14T04:18:48.544170+0000","last_fullsized":"2024-09-14T04:18:48.544170+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:21:25.648345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":241,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.452258+0000","last_change":"2024-09-14T04:10:35.385447+0000","last_active":"2024-09-14T04:18:41.452258+0000","last_peered":"2024-09-14T04:18:41.452258+0000","last_clean":"2024-09-14T04:18:41.452258+0000","last_became_active":"2024-09-14T04:10:35.385241+0000","last_became_peered":"2024-09-14T04:10:35.385241+0000","last_unstale":"2024-09-14T04:18:41.452258+0000","last_undegraded":"2024-09-14T04:18:41.452258+0000","last_fullsized":"2024-09-14T04:18:41.452258+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:02:15.709013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"289'5526","reported_seq":5546,"reported_epoch":289,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:19:01.230140+0000","last_change":"2024-09-14T04:18:17.534447+0000","last_active":"2024-09-14T04:19:01.230140+0000","last_peered":"2024-09-14T04:19:01.230140+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:18:09.382809+0000","last_became_peered":"2024-09-14T04:18:09.382809+0000","last_unstale":"2024-09-14T04:19:01.230140+0000","last_undegraded":"2024-09-14T04:18:17.534082+0000","last_fullsized":"2024-09-14T04:18:09.378221+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5526,"log_dups_size":0,"ondisk_log_size":5526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:39:35.342713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362151936,"num_objects":5526,"num_object_clones":0,"num_object_copies":16578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5526,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5526,"num_write_kb":353664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":5526}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":629,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.542242+0000","last_change":"2024-09-14T04:09:53.259468+0000","last_active":"2024-09-14T04:18:48.542242+0000","last_peered":"2024-09-14T04:18:48.542242+0000","last_clean":"2024-09-14T04:18:48.542242+0000","last_became_active":"2024-09-14T04:09:53.258842+0000","last_became_peered":"2024-09-14T04:09:53.258842+0000","last_unstale":"2024-09-14T04:18:48.542242+0000","last_undegraded":"2024-09-14T04:18:48.542242+0000","last_fullsized":"2024-09-14T04:18:48.542242+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:08:31.928142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:08:31.928142+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:52.363955+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":177,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.546482+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:18:48.546482+0000","last_peered":"2024-09-14T04:18:48.546482+0000","last_clean":"2024-09-14T04:18:48.546482+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:18:48.546482+0000","last_undegraded":"2024-09-14T04:18:48.546482+0000","last_fullsized":"2024-09-14T04:18:48.546482+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":296,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.455728+0000","last_change":"2024-09-14T04:15:33.427910+0000","last_active":"2024-09-14T04:18:41.455728+0000","last_peered":"2024-09-14T04:18:41.455728+0000","last_clean":"2024-09-14T04:15:30.398166+0000","last_became_active":"2024-09-14T04:15:33.427910+0000","last_became_peered":"2024-09-14T04:15:33.427910+0000","last_unstale":"2024-09-14T04:18:41.455728+0000","last_undegraded":"2024-09-14T04:18:41.455728+0000","last_fullsized":"2024-09-14T04:15:33.421691+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:25:55.364459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":246,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.459328+0000","last_change":"2024-09-14T03:42:35.827956+0000","last_active":"2024-09-14T04:18:41.459328+0000","last_peered":"2024-09-14T04:18:41.459328+0000","last_clean":"2024-09-14T04:18:41.459328+0000","last_became_active":"2024-09-14T03:42:35.827638+0000","last_became_peered":"2024-09-14T03:42:35.827638+0000","last_unstale":"2024-09-14T04:18:41.459328+0000","last_undegraded":"2024-09-14T04:18:41.459328+0000","last_fullsized":"2024-09-14T04:18:41.459328+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:10.064139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":274,"reported_epoch":288,"state":"active+undersized","last_fresh":"2024-09-14T04:18:41.460438+0000","last_change":"2024-09-14T04:15:33.425554+0000","last_active":"2024-09-14T04:18:41.460438+0000","last_peered":"2024-09-14T04:18:41.460438+0000","last_clean":"2024-09-14T04:15:30.423436+0000","last_became_active":"2024-09-14T04:15:33.425554+0000","last_became_peered":"2024-09-14T04:15:33.425554+0000","last_unstale":"2024-09-14T04:18:41.460438+0000","last_undegraded":"2024-09-14T04:18:41.460438+0000","last_fullsized":"2024-09-14T04:15:33.423495+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:09.544038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":268,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.460626+0000","last_change":"2024-09-14T03:42:34.647580+0000","last_active":"2024-09-14T04:18:41.460626+0000","last_peered":"2024-09-14T04:18:41.460626+0000","last_clean":"2024-09-14T04:18:41.460626+0000","last_became_active":"2024-09-14T03:42:34.647348+0000","last_became_peered":"2024-09-14T03:42:34.647348+0000","last_unstale":"2024-09-14T04:18:41.460626+0000","last_undegraded":"2024-09-14T04:18:41.460626+0000","last_fullsized":"2024-09-14T04:18:41.460626+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:05.761350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":405,"reported_epoch":289,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:18:48.545069+0000","last_change":"2024-09-14T04:16:41.496079+0000","last_active":"2024-09-14T04:18:48.545069+0000","last_peered":"2024-09-14T04:18:48.545069+0000","last_clean":"2024-09-14T04:18:48.545069+0000","last_became_active":"2024-09-14T04:16:41.495560+0000","last_became_peered":"2024-09-14T04:16:41.495560+0000","last_unstale":"2024-09-14T04:18:48.545069+0000","last_undegraded":"2024-09-14T04:18:48.545069+0000","last_fullsized":"2024-09-14T04:18:48.545069+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:36:19.683998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,1],"avail_no_missing":["0","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":299,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.538944+0000","last_change":"2024-09-14T04:06:46.355358+0000","last_active":"2024-09-14T04:18:48.538944+0000","last_peered":"2024-09-14T04:18:48.538944+0000","last_clean":"2024-09-14T04:18:48.538944+0000","last_became_active":"2024-09-14T04:06:46.334597+0000","last_became_peered":"2024-09-14T04:06:46.334597+0000","last_unstale":"2024-09-14T04:18:48.538944+0000","last_undegraded":"2024-09-14T04:18:48.538944+0000","last_fullsized":"2024-09-14T04:18:48.538944+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:08.911877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.533092+0000","last_change":"2024-09-14T03:57:55.644966+0000","last_active":"2024-09-14T04:18:48.533092+0000","last_peered":"2024-09-14T04:18:48.533092+0000","last_clean":"2024-09-14T04:18:48.533092+0000","last_became_active":"2024-09-14T03:57:55.644594+0000","last_became_peered":"2024-09-14T03:57:55.644594+0000","last_unstale":"2024-09-14T04:18:48.533092+0000","last_undegraded":"2024-09-14T04:18:48.533092+0000","last_fullsized":"2024-09-14T04:18:48.533092+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:14.793420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":222,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.452854+0000","last_change":"2024-09-14T04:07:20.038036+0000","last_active":"2024-09-14T04:18:41.452854+0000","last_peered":"2024-09-14T04:18:41.452854+0000","last_clean":"2024-09-14T04:18:41.452854+0000","last_became_active":"2024-09-14T04:07:20.037458+0000","last_became_peered":"2024-09-14T04:07:20.037458+0000","last_unstale":"2024-09-14T04:18:41.452854+0000","last_undegraded":"2024-09-14T04:18:41.452854+0000","last_fullsized":"2024-09-14T04:18:41.452854+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:41.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":173,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.543548+0000","last_change":"2024-09-14T04:10:35.377224+0000","last_active":"2024-09-14T04:18:48.543548+0000","last_peered":"2024-09-14T04:18:48.543548+0000","last_clean":"2024-09-14T04:18:48.543548+0000","last_became_active":"2024-09-14T04:10:35.376928+0000","last_became_peered":"2024-09-14T04:10:35.376928+0000","last_unstale":"2024-09-14T04:18:48.543548+0000","last_undegraded":"2024-09-14T04:18:48.543548+0000","last_fullsized":"2024-09-14T04:18:48.543548+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:22.304255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":288,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.541190+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:18:48.541190+0000","last_peered":"2024-09-14T04:18:48.541190+0000","last_clean":"2024-09-14T04:18:48.541190+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:18:48.541190+0000","last_undegraded":"2024-09-14T04:18:48.541190+0000","last_fullsized":"2024-09-14T04:18:48.541190+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":259,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.455640+0000","last_change":"2024-09-14T03:57:55.642950+0000","last_active":"2024-09-14T04:18:41.455640+0000","last_peered":"2024-09-14T04:18:41.455640+0000","last_clean":"2024-09-14T04:18:41.455640+0000","last_became_active":"2024-09-14T03:57:55.642583+0000","last_became_peered":"2024-09-14T03:57:55.642583+0000","last_unstale":"2024-09-14T04:18:41.455640+0000","last_undegraded":"2024-09-14T04:18:41.455640+0000","last_fullsized":"2024-09-14T04:18:41.455640+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:17.049613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":179,"reported_epoch":289,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:18:48.537580+0000","last_change":"2024-09-14T04:16:41.496841+0000","last_active":"2024-09-14T04:18:48.537580+0000","last_peered":"2024-09-14T04:18:48.537580+0000","last_clean":"2024-09-14T04:18:48.537580+0000","last_became_active":"2024-09-14T04:16:41.496407+0000","last_became_peered":"2024-09-14T04:16:41.496407+0000","last_unstale":"2024-09-14T04:18:48.537580+0000","last_undegraded":"2024-09-14T04:18:48.537580+0000","last_fullsized":"2024-09-14T04:18:48.537580+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:47:23.859428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,1],"avail_no_missing":["0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":174,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.546447+0000","last_change":"2024-09-14T04:10:35.387225+0000","last_active":"2024-09-14T04:18:48.546447+0000","last_peered":"2024-09-14T04:18:48.546447+0000","last_clean":"2024-09-14T04:18:48.546447+0000","last_became_active":"2024-09-14T04:10:35.386466+0000","last_became_peered":"2024-09-14T04:10:35.386466+0000","last_unstale":"2024-09-14T04:18:48.546447+0000","last_undegraded":"2024-09-14T04:18:48.546447+0000","last_fullsized":"2024-09-14T04:18:48.546447+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:33.096136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":429,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.545847+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:18:48.545847+0000","last_peered":"2024-09-14T04:18:48.545847+0000","last_clean":"2024-09-14T04:18:48.545847+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:18:48.545847+0000","last_undegraded":"2024-09-14T04:18:48.545847+0000","last_fullsized":"2024-09-14T04:18:48.545847+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":212,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:47.299343+0000","last_change":"2024-09-14T04:07:20.033936+0000","last_active":"2024-09-14T04:18:47.299343+0000","last_peered":"2024-09-14T04:18:47.299343+0000","last_clean":"2024-09-14T04:18:47.299343+0000","last_became_active":"2024-09-14T04:07:20.033082+0000","last_became_peered":"2024-09-14T04:07:20.033082+0000","last_unstale":"2024-09-14T04:18:47.299343+0000","last_undegraded":"2024-09-14T04:18:47.299343+0000","last_fullsized":"2024-09-14T04:18:47.299343+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:30:50.190772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":331,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.545312+0000","last_change":"2024-09-14T04:15:33.427023+0000","last_active":"2024-09-14T04:18:48.545312+0000","last_peered":"2024-09-14T04:18:48.545312+0000","last_clean":"2024-09-14T04:15:30.407469+0000","last_became_active":"2024-09-14T04:15:33.427023+0000","last_became_peered":"2024-09-14T04:15:33.427023+0000","last_unstale":"2024-09-14T04:18:48.545312+0000","last_undegraded":"2024-09-14T04:18:48.545312+0000","last_fullsized":"2024-09-14T04:15:33.422437+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:28:59.129487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":240,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.545526+0000","last_change":"2024-09-14T04:16:40.483586+0000","last_active":"2024-09-14T04:18:48.545526+0000","last_peered":"2024-09-14T04:18:48.545526+0000","last_clean":"2024-09-14T04:18:48.545526+0000","last_became_active":"2024-09-14T04:16:40.483231+0000","last_became_peered":"2024-09-14T04:16:40.483231+0000","last_unstale":"2024-09-14T04:18:48.545526+0000","last_undegraded":"2024-09-14T04:18:48.545526+0000","last_fullsized":"2024-09-14T04:18:48.545526+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:15.346443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":177,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.545457+0000","last_change":"2024-09-14T04:15:33.428993+0000","last_active":"2024-09-14T04:18:48.545457+0000","last_peered":"2024-09-14T04:18:48.545457+0000","last_clean":"2024-09-14T04:15:30.405877+0000","last_became_active":"2024-09-14T04:15:33.428993+0000","last_became_peered":"2024-09-14T04:15:33.428993+0000","last_unstale":"2024-09-14T04:18:48.545457+0000","last_undegraded":"2024-09-14T04:18:48.545457+0000","last_fullsized":"2024-09-14T04:15:33.422128+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:07:44.398298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":288,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:18:46.144872+0000","last_change":"2024-09-14T04:16:41.502833+0000","last_active":"2024-09-14T04:18:46.144872+0000","last_peered":"2024-09-14T04:18:46.144872+0000","last_clean":"2024-09-14T04:18:46.144872+0000","last_became_active":"2024-09-14T04:16:41.502433+0000","last_became_peered":"2024-09-14T04:16:41.502433+0000","last_unstale":"2024-09-14T04:18:46.144872+0000","last_undegraded":"2024-09-14T04:18:46.144872+0000","last_fullsized":"2024-09-14T04:18:46.144872+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:00:46.487692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":298,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.462356+0000","last_change":"2024-09-14T04:02:51.346840+0000","last_active":"2024-09-14T04:18:41.462356+0000","last_peered":"2024-09-14T04:18:41.462356+0000","last_clean":"2024-09-14T04:18:41.462356+0000","last_became_active":"2024-09-14T04:02:51.344125+0000","last_became_peered":"2024-09-14T04:02:51.344125+0000","last_unstale":"2024-09-14T04:18:41.462356+0000","last_undegraded":"2024-09-14T04:18:41.462356+0000","last_fullsized":"2024-09-14T04:18:41.462356+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:53.845061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":82,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:46.144828+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:18:46.144828+0000","last_peered":"2024-09-14T04:18:46.144828+0000","last_clean":"2024-09-14T04:18:46.144828+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:18:46.144828+0000","last_undegraded":"2024-09-14T04:18:46.144828+0000","last_fullsized":"2024-09-14T04:18:46.144828+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":298,"reported_epoch":288,"state":"active+clean","last_fresh":"2024-09-14T04:18:41.462433+0000","last_change":"2024-09-14T04:14:43.303748+0000","last_active":"2024-09-14T04:18:41.462433+0000","last_peered":"2024-09-14T04:18:41.462433+0000","last_clean":"2024-09-14T04:18:41.462433+0000","last_became_active":"2024-09-14T04:14:43.298204+0000","last_became_peered":"2024-09-14T04:14:43.298204+0000","last_unstale":"2024-09-14T04:18:41.462433+0000","last_undegraded":"2024-09-14T04:18:41.462433+0000","last_fullsized":"2024-09-14T04:18:41.462433+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:31.518529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":231,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.542845+0000","last_change":"2024-09-14T04:15:33.428329+0000","last_active":"2024-09-14T04:18:48.542845+0000","last_peered":"2024-09-14T04:18:48.542845+0000","last_clean":"2024-09-14T04:08:13.357093+0000","last_became_active":"2024-09-14T04:15:33.428329+0000","last_became_peered":"2024-09-14T04:15:33.428329+0000","last_unstale":"2024-09-14T04:18:48.542845+0000","last_undegraded":"2024-09-14T04:18:48.542845+0000","last_fullsized":"2024-09-14T04:15:33.421150+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:57:09.132723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":70,"reported_epoch":289,"state":"active+undersized","last_fresh":"2024-09-14T04:18:48.543830+0000","last_change":"2024-09-14T04:15:33.430950+0000","last_active":"2024-09-14T04:18:48.543830+0000","last_peered":"2024-09-14T04:18:48.543830+0000","last_clean":"2024-09-14T03:42:33.619286+0000","last_became_active":"2024-09-14T04:15:33.430950+0000","last_became_peered":"2024-09-14T04:15:33.430950+0000","last_unstale":"2024-09-14T04:18:48.543830+0000","last_undegraded":"2024-09-14T04:18:48.543830+0000","last_fullsized":"2024-09-14T04:15:33.423971+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:18.296447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.540102+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:18:48.540102+0000","last_peered":"2024-09-14T04:18:48.540102+0000","last_clean":"2024-09-14T04:18:48.540102+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:18:48.540102+0000","last_undegraded":"2024-09-14T04:18:48.540102+0000","last_fullsized":"2024-09-14T04:18:48.540102+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":197,"reported_epoch":289,"state":"active+clean","last_fresh":"2024-09-14T04:18:48.543697+0000","last_change":"2024-09-14T04:16:40.488129+0000","last_active":"2024-09-14T04:18:48.543697+0000","last_peered":"2024-09-14T04:18:48.543697+0000","last_clean":"2024-09-14T04:18:48.543697+0000","last_became_active":"2024-09-14T04:16:40.482286+0000","last_became_peered":"2024-09-14T04:16:40.482286+0000","last_unstale":"2024-09-14T04:18:48.543697+0000","last_undegraded":"2024-09-14T04:18:48.543697+0000","last_fullsized":"2024-09-14T04:18:48.543697+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:47:57.953684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5855444992,"num_objects":89347,"num_object_clones":0,"num_object_copies":268041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50357,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89347,"num_write_kb":5718208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14349107200,"data_stored":14349107200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89347,"ondisk_log_size":89347,"up":39,"acting":39,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":1528,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18792,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":87,"acting":88,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":21,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":29,"acting":29,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201741,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4293588,"kb_used_data":3265740,"kb_used_omap":1,"kb_used_meta":1027838,"kb_avail":89451564,"statfs":{"total":95995035648,"available":91598401536,"internally_reserved":0,"allocated":3344117760,"data_stored":3342244658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1052506571},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463379,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963720,"kb_used_data":1098304,"kb_used_omap":1,"kb_used_meta":865406,"kb_avail":91781432,"statfs":{"total":95995035648,"available":93984186368,"internally_reserved":0,"allocated":1124663296,"data_stored":1122802398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":886176199},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725015,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2245512,"kb_used_data":1083136,"kb_used_omap":1,"kb_used_meta":1162366,"kb_avail":91499640,"statfs":{"total":95995035648,"available":93695631360,"internally_reserved":0,"allocated":1109131264,"data_stored":1107266446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1190263238},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986651,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3163984,"kb_used_data":2136136,"kb_used_omap":1,"kb_used_meta":1027838,"kb_avail":90581168,"statfs":{"total":95995035648,"available":92755116032,"internally_reserved":0,"allocated":2187403264,"data_stored":2185533949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1052506566},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748680,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3513924,"kb_used_data":2852988,"kb_used_omap":4,"kb_used_meta":660923,"kb_avail":90231228,"statfs":{"total":95995035648,"available":92396777472,"internally_reserved":0,"allocated":2921459712,"data_stored":2919598702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":676785708},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912164,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3074656,"kb_used_data":2703128,"kb_used_omap":6,"kb_used_meta":371513,"kb_avail":90670496,"statfs":{"total":95995035648,"available":92846587904,"internally_reserved":0,"allocated":2768003072,"data_stored":2766371862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":380429881},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468223,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2192124,"kb_used_data":1425780,"kb_used_omap":5,"kb_used_meta":766330,"kb_avail":91553028,"statfs":{"total":95995035648,"available":93750300672,"internally_reserved":0,"allocated":1459998720,"data_stored":1458143867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":784722271},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342338,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2475404,"kb_used_data":2074820,"kb_used_omap":10,"kb_used_meta":400565,"kb_avail":91269748,"statfs":{"total":95995035648,"available":93460221952,"internally_reserved":0,"allocated":2124615680,"data_stored":2122746501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":410178770},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2132934656,"data_stored":2132934656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1467875328,"data_stored":1467875328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2940207104,"data_stored":2940207104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2205286400,"data_stored":2205286400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1117913088,"data_stored":1117913088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1132134400,"data_stored":1132134400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3352756224,"data_stored":3352756224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:19:05.351 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:19:05.351 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-09-14T04:19:05.357 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:19:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:05.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:05.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:05.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:05.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 95959 95943 124.909 84.9375 0.00976592 0.00800125 2024-09-14T04:19:05.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:05.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:05 smithi001 ceph-mon[107399]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:19:05.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:05.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:05 smithi001 ceph-mon[112165]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:19:05.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:05.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:05.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:05.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:05 smithi012 ceph-mon[101180]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:19:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:05.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:05.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:05.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:06.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:06.071 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:06.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:06.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:06.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:06.358 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:06.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:06.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:06.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 97281 97265 124.046 82.625 0.00561093 0.00805682 2024-09-14T04:19:06.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:06.640 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:06.656 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:06 smithi001 ceph-mon[112165]: pgmap v1055: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 51276/274794 objects degraded (18.660%) 2024-09-14T04:19:06.656 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:06 smithi001 ceph-mon[107399]: pgmap v1055: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 51276/274794 objects degraded (18.660%) 2024-09-14T04:19:06.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:06 smithi012 ceph-mon[101180]: pgmap v1055: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 51276/274794 objects degraded (18.660%) 2024-09-14T04:19:06.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:06.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:06.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:06.912 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:06.955 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:19:06.955 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:19:06.957 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:06] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-09-14T04:19:07.012 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:07.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:07.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:07.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:07.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:07.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:07.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:07.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:07.456 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:19:07.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 98521 98505 123.115 77.5 0.0140754 0.00811802 2024-09-14T04:19:07.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:07.585 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:07.585 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:07.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:07.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:07.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:07.747 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:07.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:07.867 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:07.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:07.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:08.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:08.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:19:08.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:08.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:08.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:08.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:08.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:08.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:08.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:08.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 99860 99844 122.342 83.6875 0.00507375 0.00816885 2024-09-14T04:19:08.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:08.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:08.705 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[107399]: pgmap v1056: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 54243/288834 objects degraded (18.780%) 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[107399]: osdmap e290: 8 total, 7 up, 8 in 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[112165]: pgmap v1056: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 54243/288834 objects degraded (18.780%) 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:08 smithi001 ceph-mon[112165]: osdmap e290: 8 total, 7 up, 8 in 2024-09-14T04:19:08.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:08.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:08 smithi012 ceph-mon[101180]: pgmap v1056: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 54243/288834 objects degraded (18.780%) 2024-09-14T04:19:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:08 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:19:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3462031998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:08 smithi012 ceph-mon[101180]: osdmap e290: 8 total, 7 up, 8 in 2024-09-14T04:19:08.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:08.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:08.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:09.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:09.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:09.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:09.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:09.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:09.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:09.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:09.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 101202 101186 121.602 83.875 0.0142853 0.00821884 2024-09-14T04:19:09.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:09.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:09.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:09.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 54243/288834 objects degraded (18.780%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 54243/288834 objects degraded (18.780%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:09.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:09.836 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 54243/288834 objects degraded (18.780%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:09.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:09.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:09.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:10.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.115 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:10.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:10.168 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:10.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:10.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:10.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.389 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:10.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:10.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:10.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:10.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 102440 102424 120.767 77.375 0.00665161 0.00827522 2024-09-14T04:19:10.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:10.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:10 smithi001 ceph-mon[107399]: pgmap v1058: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 55052/294912 objects degraded (18.667%) 2024-09-14T04:19:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:10 smithi001 ceph-mon[112165]: pgmap v1058: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 55052/294912 objects degraded (18.667%) 2024-09-14T04:19:10.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:10.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:10 smithi012 ceph-mon[101180]: pgmap v1058: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 55052/294912 objects degraded (18.667%) 2024-09-14T04:19:10.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:10.934 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:10.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:11.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:11.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:11.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:11.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:11.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:11.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:11.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:11.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:11.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:11.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 103554 103538 119.82 69.625 0.0184143 0.00834113 2024-09-14T04:19:11.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:11.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:11.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:11.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:11.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:11.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:11.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:11.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:12.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:12.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:12.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:12.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:12.178 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:12.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:19:12.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:12.314 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:12.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:12.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:12.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:12.484 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:12 smithi001 ceph-mon[112165]: pgmap v1059: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 56637/302031 objects degraded (18.752%) 2024-09-14T04:19:12.484 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:12 smithi001 ceph-mon[107399]: pgmap v1059: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 56637/302031 objects degraded (18.752%) 2024-09-14T04:19:12.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 104819 104803 119.079 79.0625 0.0259978 0.00839284 2024-09-14T04:19:12.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:12.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:12.657 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:19:12.657 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:19:12.694 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:19:12.694 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:19:12.695 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:19:12.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:12.725 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:19:12.725 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:19:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:12.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:12.750 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:19:12.750 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:19:12.772 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:19:12.772 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:19:12.800 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:19:12.800 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:19:12.833 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:19:12.833 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:19:12.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:12 smithi012 ceph-mon[101180]: pgmap v1059: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 56637/302031 objects degraded (18.752%) 2024-09-14T04:19:12.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:12.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:12.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:12.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:13.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:13.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:13.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:13.264 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:13.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:13.399 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:13.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:13.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:13.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 106083 106067 118.363 79 0.0104894 0.008444 2024-09-14T04:19:13.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:13.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:13.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:13.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:13 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3588255952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:13.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:13.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:13.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:13 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3588255952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:13 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3588255952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:13.991 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:13.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:14.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:14.130 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:14.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:14.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:14.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:14.418 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:14.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:14.536 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 107355 107339 117.681 79.5 0.00699605 0.00849313 2024-09-14T04:19:14.556 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:14.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:14.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:14 smithi001 ceph-mon[107399]: pgmap v1060: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 57792/307932 objects degraded (18.768%) 2024-09-14T04:19:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 57792/307932 objects degraded (18.768%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:14 smithi001 ceph-mon[112165]: pgmap v1060: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 57792/307932 objects degraded (18.768%) 2024-09-14T04:19:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 57792/307932 objects degraded (18.768%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:14.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:14.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:14 smithi012 ceph-mon[101180]: pgmap v1060: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 57792/307932 objects degraded (18.768%) 2024-09-14T04:19:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 57792/307932 objects degraded (18.768%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:14.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:14.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:14.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:15.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:15.096 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:15.097 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:19:15.105 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:19:15.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:15.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:15.271 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:15.373 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:15.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:15.490 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 1 pg_num to 21 2024-09-14T04:19:15.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:15.538 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 108567 108551 116.955 75.75 0.0194087 0.00854509 2024-09-14T04:19:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:15.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:15.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:15.680 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:15.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3992160974' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:15 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3992160974' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:15 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.782 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:15.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3992160974' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:15 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:19:15.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:15.956 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:15.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:16.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:16.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:16.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:16.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:16.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:16.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:16.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:16.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:16.538 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 109839 109823 116.32 79.5 0.00960946 0.00859247 2024-09-14T04:19:16.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:16.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:16.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:16.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:16 smithi012 ceph-mon[101180]: pgmap v1061: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 58597/313749 objects degraded (18.676%) 2024-09-14T04:19:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:16 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T04:19:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:16 smithi012 ceph-mon[101180]: osdmap e291: 8 total, 7 up, 8 in 2024-09-14T04:19:16.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:16.911 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:16.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:16] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-09-14T04:19:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[107399]: pgmap v1061: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 58597/313749 objects degraded (18.676%) 2024-09-14T04:19:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T04:19:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[107399]: osdmap e291: 8 total, 7 up, 8 in 2024-09-14T04:19:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[112165]: pgmap v1061: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 58597/313749 objects degraded (18.676%) 2024-09-14T04:19:16.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T04:19:16.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:16 smithi001 ceph-mon[112165]: osdmap e291: 8 total, 7 up, 8 in 2024-09-14T04:19:17.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:17.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:17.106 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:19:17.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:17.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:17.285 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:17.317 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:17.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:17.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:17.538 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:19:17.538299+0000 min lat: 0.000960706 max lat: 0.455133 avg lat: 0.00864849 2024-09-14T04:19:17.538 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:19:17.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 15 110971 110956 115.562 70.8125 0.0118255 0.00864849 2024-09-14T04:19:17.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:17.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:17.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:17.736 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:17.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:17.838 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T04:19:17.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:17.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:17.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T04:19:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T04:19:17.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:18.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:18.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:18.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:18.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:18.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:18.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:18.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:18.538 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 112273 112257 115 81.3125 0.0119126 0.00869073 2024-09-14T04:19:18.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:18.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:18.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:18.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:18.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[107399]: pgmap v1063: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 61445/326817 objects degraded (18.801%) 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[107399]: osdmap e292: 8 total, 7 up, 8 in 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[112165]: pgmap v1063: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 61445/326817 objects degraded (18.801%) 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T04:19:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:18 smithi001 ceph-mon[112165]: osdmap e292: 8 total, 7 up, 8 in 2024-09-14T04:19:18.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:18 smithi012 ceph-mon[101180]: pgmap v1063: 99 pgs: 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 61445/326817 objects degraded (18.801%) 2024-09-14T04:19:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T04:19:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:18 smithi012 ceph-mon[101180]: osdmap e292: 8 total, 7 up, 8 in 2024-09-14T04:19:18.848 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:18.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:18.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:18.987 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:18.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:19.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:19.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:19.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:19.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:19.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:19.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:19.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:19.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:19.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 113559 113543 114.442 80.375 0.00393104 0.00873313 2024-09-14T04:19:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:19.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:19.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:19.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:19.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.820 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:19.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:19.835 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: osdmap e293: 8 total, 7 up, 8 in 2024-09-14T04:19:19.835 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.7 scrub starts 2024-09-14T04:19:19.835 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.7 scrub ok 2024-09-14T04:19:19.835 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.b scrub starts 2024-09-14T04:19:19.835 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.b scrub ok 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 61445/326817 objects degraded (18.801%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.6 scrub starts 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[107399]: 1.6 scrub ok 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: osdmap e293: 8 total, 7 up, 8 in 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.7 scrub starts 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.7 scrub ok 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.b scrub starts 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.b scrub ok 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 61445/326817 objects degraded (18.801%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:19.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.6 scrub starts 2024-09-14T04:19:19.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:19 smithi001 ceph-mon[112165]: 1.6 scrub ok 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: osdmap e293: 8 total, 7 up, 8 in 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.7 scrub starts 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.7 scrub ok 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.b scrub starts 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.b scrub ok 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 61445/326817 objects degraded (18.801%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.6 scrub starts 2024-09-14T04:19:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:19 smithi012 ceph-mon[101180]: 1.6 scrub ok 2024-09-14T04:19:19.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:19.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:19.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:20.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:20.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:20.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:20.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:20.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:20.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:20.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:20.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:20.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 114829 114813 113.885 79.375 0.0190952 0.00877614 2024-09-14T04:19:20.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:20.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:20.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:20.749 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85146 10.2 10:4e8e28f3:::benchmark_data_smithi001_190881_object85145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 571.53877689199999, 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1709060940000002, 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.750 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85146 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207173+0000", 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207175+0000", 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.813e-06 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207199+0000", 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4312999999999999e-05 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.752 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207216+0000", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6694999999999999e-05 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207223+0000", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.08e-06 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207276+0000", 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3038999999999998e-05 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.753 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207427+0000", 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015100000000000001 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.207565+0000", 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013732699999999999 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.754 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223766+0000", 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016201706 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223775+0000", 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5890000000000003e-06 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.755 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214581+0000", 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99080585499999996 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214598+0000", 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7051e-05 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.756 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378012+0000", 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1634141140000001 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.378079+0000", 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7512e-05 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.757 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85153 10.2 10:4ace6224:::benchmark_data_smithi001_190881_object85152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 571.52436645299997, 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1713794549999998, 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.758 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85153 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221584+0000", 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.759 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221586+0000", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9290000000000001e-06 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221614+0000", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8419999999999999e-05 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221629+0000", 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4602e-05 2024-09-14T04:19:20.760 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.221638+0000", 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8959999999999992e-06 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223951+0000", 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023134240000000001 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.761 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.223958+0000", 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.754e-06 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214629+0000", 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.990671623 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214640+0000", 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.049e-05 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.762 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379336+0000", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1646960749999999 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379484+0000", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014773 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.379590+0000", 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106259 2024-09-14T04:19:20.763 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384020+0000", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044296270000000002 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384076+0000", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6487999999999999e-05 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390537+0000", 2024-09-14T04:19:20.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064613240000000001 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.390591+0000", 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3164999999999997e-05 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.392963+0000", 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023726490000000001 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.765 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:85211 10.2 10:452ef993:::benchmark_data_smithi001_190881_object85210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 571.46775041599994, 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1165732400000001, 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 85211 2024-09-14T04:19:20.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278200+0000", 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278202+0000", 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9640000000000002e-06 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278222+0000", 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9796000000000002e-05 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278234+0000", 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1922999999999999e-05 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278242+0000", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3000000000000002e-06 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278320+0000", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8647999999999996e-05 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:49.278333+0000", 2024-09-14T04:19:20.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.257e-05 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214669+0000", 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.936335849 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:50.214677+0000", 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1559999999999995e-06 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381729+0000", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.167051839 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381867+0000", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013809400000000001 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.381978+0000", 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011068 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.384104+0000", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002125886 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.386784+0000", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002680038 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393032+0000", 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062480950000000004 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.393073+0000", 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1409999999999998e-05 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:09:51.394773+0000", 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001699992 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.773 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:157894 10.4 10:2bdef4ea:::benchmark_data_smithi001_190881_object42754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 473.36847723300002, 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.14587671099999999, 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.774 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 157894 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377473+0000", 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377474+0000", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7199999999999997e-07 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377479+0000", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1019999999999996e-06 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377491+0000", 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2379e-05 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.377498+0000", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.849e-06 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.408602+0000", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031104053 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.409004+0000", 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040173900000000002 2024-09-14T04:19:20.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.409491+0000", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048706299999999998 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.439679+0000", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030187535000000001 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.439712+0000", 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.3433000000000003e-05 2024-09-14T04:19:20.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.780 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.448849+0000", 2024-09-14T04:19:20.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0091365809999999995 2024-09-14T04:19:20.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.518170+0000", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.069321764999999994 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.518497+0000", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032694199999999999 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:11:27.523350+0000", 2024-09-14T04:19:20.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004852398 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:211990 10.4 10:28009dd3:::benchmark_data_smithi001_190881_object96850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 401.57304574, 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21457178399999999, 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.782 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 211990 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:19:20.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172905+0000", 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5499999999999997e-07 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172909+0000", 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9349999999999996e-06 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172918+0000", 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.149e-06 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.172925+0000", 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4790000000000001e-06 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.177688+0000", 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004763321 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.177879+0000", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019074200000000001 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.178050+0000", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017125800000000001 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.182884+0000", 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00483411 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.188759+0000", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058751059999999997 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220023+0000", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031263460999999999 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.385148+0000", 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.16512544300000001 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.385215+0000", 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6970000000000004e-05 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.387476+0000", 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022610550000000001 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212006 10.4 10:20e112b0:::benchmark_data_smithi001_190881_object96866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 401.554499679, 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21726297, 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212006 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191451+0000", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191452+0000", 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.728e-06 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191458+0000", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8810000000000001e-06 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191474+0000", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5364999999999999e-05 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191482+0000", 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9650000000000008e-06 2024-09-14T04:19:20.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191550+0000", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8058999999999993e-05 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191713+0000", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000163497 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.191870+0000", 2024-09-14T04:19:20.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015658499999999999 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.195822+0000", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039520839999999998 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.198227+0000", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002405084 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220065+0000", 2024-09-14T04:19:20.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021838446000000001 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.403595+0000", 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18352981800000001 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.404044+0000", 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00044926699999999999 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408714+0000", 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0046691909999999996 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212011 10.4 10:2e1943d4:::benchmark_data_smithi001_190881_object96871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 401.545497764, 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21107719999999999, 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.795 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212011 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200452+0000", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200454+0000", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.587e-06 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200459+0000", 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4680000000000003e-06 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200471+0000", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2289e-05 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200476+0000", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4299999999999997e-06 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200535+0000", 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8785e-05 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200717+0000", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000181955 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.200904+0000", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000187061 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208534+0000", 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007629767 2024-09-14T04:19:20.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208551+0000", 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7577000000000001e-05 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220113+0000", 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011561235 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408793+0000", 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.188680184 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.408872+0000", 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9589e-05 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411530+0000", 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026572729999999999 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212013 10.4 10:2eef1eae:::benchmark_data_smithi001_190881_object96873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 401.54279836900002, 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21087093100000001, 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212013 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.802 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203152+0000", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203154+0000", 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2019999999999998e-06 2024-09-14T04:19:20.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203161+0000", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203175+0000", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4576000000000001e-05 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.203184+0000", 2024-09-14T04:19:20.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1259999999999998e-06 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204557+0000", 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001373785 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204722+0000", 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000165119 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.204910+0000", 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000187319 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208584+0000", 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003674053 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.208599+0000", 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5474000000000001e-05 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.220157+0000", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011557507 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411647+0000", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.19149049900000001 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.411710+0000", 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2779000000000003e-05 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414023+0000", 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023126840000000002 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45035.0:212022 10.4 10:2ef778a0:::benchmark_data_smithi001_190881_object96882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 401.51673519299999, 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18721781100000001, 2024-09-14T04:19:20.808 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45035", 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 212022 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229215+0000", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229217+0000", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8169999999999999e-06 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229223+0000", 2024-09-14T04:19:20.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4989999999999999e-06 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229240+0000", 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6529999999999999e-05 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229247+0000", 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1829999999999997e-06 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229299+0000", 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2008000000000001e-05 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229414+0000", 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011464 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3,4", 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.229519+0000", 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105183 2024-09-14T04:19:20.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.232915+0000", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033962390000000001 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.233109+0000", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019340500000000001 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.237422+0000", 2024-09-14T04:19:20.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043136060000000002 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414077+0000", 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.17665492199999999 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.414133+0000", 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5883e-05 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:12:39.416433+0000", 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022998960000000001 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:26525 11.0 11:039722ec:::benchmark_data_smithi001_228075_object26524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 300.48553094900001, 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18390612100000001, 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.816 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 26525 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260419+0000", 2024-09-14T04:19:20.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260421+0000", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3990000000000001e-06 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260444+0000", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3757999999999999e-05 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260462+0000", 2024-09-14T04:19:20.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7589999999999999e-05 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260470+0000", 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6329999999999996e-06 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260525+0000", 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5046000000000001e-05 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260653+0000", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012794799999999999 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260882+0000", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022941300000000001 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263376+0000", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024943909999999999 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263962+0000", 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00058602100000000005 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444206+0000", 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18024396200000001 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444271+0000", 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4189000000000005e-05 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444325+0000", 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4771000000000001e-05 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:26527 11.0 11:09da525d:::benchmark_data_smithi001_228075_object26526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 300.48521491399998, 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18375396499999999, 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.822 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 26527 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260735+0000", 2024-09-14T04:19:20.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260736+0000", 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7400000000000002e-07 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260763+0000", 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7183999999999999e-05 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260777+0000", 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3246e-05 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.260783+0000", 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6660000000000002e-06 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262177+0000", 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001394045 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262318+0000", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014096900000000001 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.262488+0000", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016987799999999999 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.263718+0000", 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012298980000000001 2024-09-14T04:19:20.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.265281+0000", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001562995 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444390+0000", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.17910886600000001 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444451+0000", 2024-09-14T04:19:20.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0761000000000003e-05 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:14:20.444489+0000", 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8583e-05 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111729 11.c 11:38344d7b:::benchmark_data_smithi001_228075_object111728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:19:20.828 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 228.348490381, 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0573153820000001, 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111729 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8099999999999998e-07 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397510+0000", 2024-09-14T04:19:20.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0049999999999997e-05 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397517+0000", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9469999999999997e-06 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397522+0000", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9030000000000003e-06 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410401+0000", 2024-09-14T04:19:20.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012879214 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410536+0000", 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013432900000000001 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410682+0000", 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014610799999999999 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420457+0000", 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097754369999999997 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432471+0000", 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012013348 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438812+0000", 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063412800000000004 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438822+0000", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0463e-05 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454678+0000", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0158558950000001 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454775+0000", 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7027000000000005e-05 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.834 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111734 11.c 11:3256a745:::benchmark_data_smithi001_228075_object111733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 228.341465793, 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0505815119999999, 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:19:20.835 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: pgmap v1066: 109 pgs: 10 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 62233/332691 objects degraded (18.706%) 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.14 scrub starts 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.14 scrub ok 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.3 scrub starts 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.3 scrub ok 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:19:20.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[107399]: 1.e scrub ok 2024-09-14T04:19:20.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: pgmap v1066: 109 pgs: 10 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 62233/332691 objects degraded (18.706%) 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.14 scrub starts 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.14 scrub ok 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.3 scrub starts 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.3 scrub ok 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:19:20.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:20 smithi001 ceph-mon[112165]: 1.e scrub ok 2024-09-14T04:19:20.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:20.838 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.838 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111734 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404486+0000", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.375e-06 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404540+0000", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4305000000000003e-05 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404552+0000", 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2058999999999999e-05 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404560+0000", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5070000000000003e-06 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420484+0000", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015924223000000001 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420624+0000", 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000139889 2024-09-14T04:19:20.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420770+0000", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014612800000000001 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432477+0000", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011706658999999999 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438869+0000", 2024-09-14T04:19:20.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006392516 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438876+0000", 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1420000000000004e-06 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454995+0000", 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161184889999999 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455066+0000", 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1219999999999999e-05 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111737 11.c 11:30a256cb:::benchmark_data_smithi001_228075_object111736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 228.340965498, 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.072352075, 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:19:20.844 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111737 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:19:20.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404986+0000", 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5600000000000004e-07 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405008+0000", 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2568000000000001e-05 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405023+0000", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4442e-05 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405028+0000", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1789999999999999e-06 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427311+0000", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022282885999999998 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427456+0000", 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014496299999999999 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.428041+0000", 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00058510400000000003 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438907+0000", 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010866310000000001 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438913+0000", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455100+0000", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161871069999999 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for degraded object", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455507+0000", 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040602299999999999 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477138+0000", 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021631583999999999 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477337+0000", 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000198717 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111746 11.c 11:38d82d32:::benchmark_data_smithi001_228075_object111745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:19:20.850 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 228.33188035699999, 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0634765070000001, 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111746 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:19:20.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414071+0000", 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.519e-06 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414094+0000", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2546999999999999e-05 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414108+0000", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4425e-05 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414117+0000", 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999992e-06 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438937+0000", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024819529 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438942+0000", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1930000000000001e-06 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455547+0000", 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016605078 2024-09-14T04:19:20.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455715+0000", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001685 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455935+0000", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000219936 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477381+0000", 2024-09-14T04:19:20.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021446094999999998 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477395+0000", 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3925e-05 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477458+0000", 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2985999999999996e-05 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477546+0000", 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8023999999999996e-05 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111759 11.c 11:3f216db5:::benchmark_data_smithi001_228075_object111758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 228.30305782600001, 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0348463530000001, 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.857 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111759 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442895+0000", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4980000000000001e-06 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442917+0000", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2342999999999999e-05 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442932+0000", 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4917e-05 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442940+0000", 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.892e-06 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443001+0000", 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1416999999999998e-05 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443012+0000", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.092e-05 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459584+0000", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0165711500000001 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459713+0000", 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012947499999999999 2024-09-14T04:19:20.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459870+0000", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015698099999999999 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477555+0000", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017684921999999999 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477612+0000", 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7117000000000002e-05 2024-09-14T04:19:20.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477662+0000", 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9512000000000003e-05 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477739+0000", 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7208999999999999e-05 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170311 11.0 11:0a21328e:::benchmark_data_smithi001_228075_object51996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 166.80703430599999, 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.096979240999999994, 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170311 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938916+0000", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938917+0000", 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.184e-06 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938922+0000", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9370000000000003e-06 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938934+0000", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1569999999999999e-05 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938940+0000", 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5939999999999999e-06 2024-09-14T04:19:20.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.938991+0000", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0785999999999998e-05 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939081+0000", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0138999999999999e-05 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939147+0000", 2024-09-14T04:19:20.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6026999999999997e-05 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.945935+0000", 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0067877959999999996 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.959189+0000", 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013253699000000001 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035759+0000", 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.076570605999999999 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035846+0000", 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6504999999999999e-05 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.035895+0000", 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9397999999999997e-05 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170312 11.0 11:088ab68c:::benchmark_data_smithi001_228075_object51997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 166.80599234600001, 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.103849861, 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.870 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170312 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939958+0000", 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939959+0000", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2399999999999996e-07 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939962+0000", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0910000000000001e-06 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939973+0000", 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1144999999999999e-05 2024-09-14T04:19:20.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.939980+0000", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2779999999999996e-06 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942366+0000", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023860040000000002 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942501+0000", 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013510700000000001 2024-09-14T04:19:20.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.942608+0000", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010642299999999999 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.946040+0000", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003432189 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.963603+0000", 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017562782999999998 2024-09-14T04:19:20.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043663+0000", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080059882999999998 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043758+0000", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5027999999999999e-05 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043808+0000", 2024-09-14T04:19:20.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0006000000000002e-05 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:170322 11.0 11:03b3ce39:::benchmark_data_smithi001_228075_object52007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 166.78957193900001, 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.087605957999999998, 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.876 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 170322 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956378+0000", 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956380+0000", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2720000000000001e-06 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956384+0000", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7319999999999998e-06 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956393+0000", 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.789e-06 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956398+0000", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1590000000000001e-06 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956440+0000", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1471000000000003e-05 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956534+0000", 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3986000000000004e-05 2024-09-14T04:19:20.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.956604+0000", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9989999999999999e-05 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.959203+0000", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025993069999999999 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:33.963657+0000", 2024-09-14T04:19:20.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044537209999999999 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043860+0000", 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.080202829000000003 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043943+0000", 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3700000000000002e-05 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:16:34.043984+0000", 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1001999999999998e-05 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:236467 11.0 11:0f18aa00:::benchmark_data_smithi001_228075_object118152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:17:48.294398+0000", 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 92.451552567999997, 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.087792588000000005, 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:19:20.882 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 236467 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294398+0000", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294398+0000", 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:19:20.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294400+0000", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0370000000000001e-06 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294405+0000", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2129999999999999e-06 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294417+0000", 2024-09-14T04:19:20.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2101e-05 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294425+0000", 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8359999999999992e-06 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294473+0000", 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.808e-05 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:19:20.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294595+0000", 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012186200000000001 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.294732+0000", 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000137608 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.297620+0000", 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028875260000000001 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.300977+0000", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033573930000000002 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.382102+0000", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.081124326999999996 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.382148+0000", 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6776000000000001e-05 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:19:20.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:17:48.382190+0000", 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1829000000000002e-05 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:19:20.888 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: pgmap v1066: 109 pgs: 10 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 66 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 62233/332691 objects degraded (18.706%) 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.14 scrub starts 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.14 scrub ok 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.3 scrub starts 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.3 scrub ok 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:19:20.889 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:20 smithi012 ceph-mon[101180]: 1.e scrub ok 2024-09-14T04:19:20.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:20.975 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:20.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:20.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:21.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:21.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:21.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:21.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:21.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:21.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:21.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:21.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:21.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 116043 116027 113.291 75.875 0.0175116 0.00882188 2024-09-14T04:19:21.539 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.5 scrub starts 2024-09-14T04:19:21.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.5 scrub starts 2024-09-14T04:19:21.539 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.5 scrub ok 2024-09-14T04:19:21.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:21.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:21.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.2 scrub starts 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.2 scrub ok 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.4 scrub starts 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.4 scrub ok 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.13 scrub starts 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[107399]: 1.13 scrub ok 2024-09-14T04:19:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.5 scrub ok 2024-09-14T04:19:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.2 scrub starts 2024-09-14T04:19:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.2 scrub ok 2024-09-14T04:19:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.4 scrub starts 2024-09-14T04:19:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.4 scrub ok 2024-09-14T04:19:21.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.13 scrub starts 2024-09-14T04:19:21.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:21 smithi001 ceph-mon[112165]: 1.13 scrub ok 2024-09-14T04:19:21.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:21.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.5 scrub starts 2024-09-14T04:19:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.5 scrub ok 2024-09-14T04:19:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.2 scrub starts 2024-09-14T04:19:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.2 scrub ok 2024-09-14T04:19:21.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.4 scrub starts 2024-09-14T04:19:21.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.4 scrub ok 2024-09-14T04:19:21.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.13 scrub starts 2024-09-14T04:19:21.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:21 smithi012 ceph-mon[101180]: 1.13 scrub ok 2024-09-14T04:19:21.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:21.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:21.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:21.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:22.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:22.092 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:22.194 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:22.227 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:22.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:22.362 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:22.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:22.502 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:22.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 117335 117319 112.79 80.75 0.0125353 0.00886121 2024-09-14T04:19:22.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:22.641 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:22.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:22.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:22 smithi012 ceph-mon[101180]: pgmap v1067: 109 pgs: 4 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 72 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 63746/339612 objects degraded (18.770%) 2024-09-14T04:19:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:22 smithi012 ceph-mon[101180]: 1.d scrub starts 2024-09-14T04:19:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:22 smithi012 ceph-mon[101180]: 1.d scrub ok 2024-09-14T04:19:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:22 smithi012 ceph-mon[101180]: 1.f scrub starts 2024-09-14T04:19:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:22 smithi012 ceph-mon[101180]: 1.f scrub ok 2024-09-14T04:19:22.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:22.904 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[112165]: pgmap v1067: 109 pgs: 4 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 72 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 63746/339612 objects degraded (18.770%) 2024-09-14T04:19:22.904 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[112165]: 1.d scrub starts 2024-09-14T04:19:22.904 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[112165]: 1.d scrub ok 2024-09-14T04:19:22.904 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[112165]: 1.f scrub starts 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[112165]: 1.f scrub ok 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[107399]: pgmap v1067: 109 pgs: 4 unknown, 3 active+clean+remapped, 18 active+undersized, 12 active+undersized+degraded, 72 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 63746/339612 objects degraded (18.770%) 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[107399]: 1.d scrub starts 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[107399]: 1.d scrub ok 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[107399]: 1.f scrub starts 2024-09-14T04:19:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:22 smithi001 ceph-mon[107399]: 1.f scrub ok 2024-09-14T04:19:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:22.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:22.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:22.990 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:23.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:23.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:23.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:23.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:23.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:23.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:23.338 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:23.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:23.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:23.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:19:23.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 118609 118593 112.287 79.625 0.00974121 0.00890092 2024-09-14T04:19:23.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:23.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[112165]: 1.0 scrub starts 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[112165]: 1.0 scrub ok 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[112165]: 1.c scrub starts 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[112165]: 1.c scrub ok 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[112165]: pgmap v1068: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 64884/345468 objects degraded (18.781%) 2024-09-14T04:19:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[107399]: 1.0 scrub starts 2024-09-14T04:19:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[107399]: 1.0 scrub ok 2024-09-14T04:19:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[107399]: 1.c scrub starts 2024-09-14T04:19:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[107399]: 1.c scrub ok 2024-09-14T04:19:23.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:23 smithi001 ceph-mon[107399]: pgmap v1068: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 64884/345468 objects degraded (18.781%) 2024-09-14T04:19:23.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:23.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:23 smithi012 ceph-mon[101180]: 1.0 scrub starts 2024-09-14T04:19:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:23 smithi012 ceph-mon[101180]: 1.0 scrub ok 2024-09-14T04:19:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:23 smithi012 ceph-mon[101180]: 1.c scrub starts 2024-09-14T04:19:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:23 smithi012 ceph-mon[101180]: 1.c scrub ok 2024-09-14T04:19:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:23 smithi012 ceph-mon[101180]: pgmap v1068: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 64884/345468 objects degraded (18.781%) 2024-09-14T04:19:23.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:23.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:23.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:24.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:24.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:24.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:24.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:24.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:24.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:24.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:24.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:24.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:24.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:24.539 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 119879 119863 111.796 79.375 0.0060744 0.00894084 2024-09-14T04:19:24.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:24.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:24.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 64884/345468 objects degraded (18.781%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 64884/345468 objects degraded (18.781%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:24.741 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:24.743 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:24.752 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2024-09-14T04:19:24.753 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:19:24.753 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2024-09-14T04:19:24.753 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2024-09-14T04:19:24.753 DEBUG:teuthology.orchestra.run.smithi001:osd.2> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service 2024-09-14T04:19:24.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl start ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2 2024-09-14T04:19:24.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 64884/345468 objects degraded (18.781%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:24.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:24.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:24.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:25.014 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:25.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:25.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:25.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:24 smithi001 systemd[1]: Starting Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:19:25.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:25.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:25.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:25.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:25.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:25.540 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 121156 121140 111.323 79.8125 0.0050151 0.00897804 2024-09-14T04:19:25.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:25.564 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:19:25.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:25.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:19:25.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:25 smithi001 ceph-mon[112165]: pgmap v1069: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 65668/351468 objects degraded (18.684%) 2024-09-14T04:19:25.687 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:19:25.687 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:25 smithi001 ceph-mon[107399]: pgmap v1069: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 65668/351468 objects degraded (18.684%) 2024-09-14T04:19:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.704 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:25.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:25.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:25.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:19:25.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:25 smithi012 ceph-mon[101180]: pgmap v1069: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 65668/351468 objects degraded (18.684%) 2024-09-14T04:19:25.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:25.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:25.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:25.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:25.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:26.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:26.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:26.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:26.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:26.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:26.373 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:26.411 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:26.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:26.541 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 122424 122408 110.857 79.25 0.010711 0.00901629 2024-09-14T04:19:26.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:26.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:26.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:19:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[107399]: osdmap e294: 8 total, 7 up, 8 in 2024-09-14T04:19:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:26.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:19:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-mon[112165]: osdmap e294: 8 total, 7 up, 8 in 2024-09-14T04:19:26.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:26.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:26.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:26 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/156900105' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:19:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:26 smithi012 ceph-mon[101180]: osdmap e294: 8 total, 7 up, 8 in 2024-09-14T04:19:26.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:26.957 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:26.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:26.961 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:26] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.51.0" 2024-09-14T04:19:27.059 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:27.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:27.200 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:27.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:27.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:27.366 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:27.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:27.502 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:27.541 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 123592 123576 110.317 73 0.00954728 0.0090605 2024-09-14T04:19:27.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:27.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:27.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:27.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:27 smithi012 ceph-mon[101180]: pgmap v1071: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 68418/364458 objects degraded (18.773%) 2024-09-14T04:19:27.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:27.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:27.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:27 smithi001 ceph-mon[107399]: pgmap v1071: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 68418/364458 objects degraded (18.773%) 2024-09-14T04:19:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:27 smithi001 ceph-mon[112165]: pgmap v1071: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 68418/364458 objects degraded (18.773%) 2024-09-14T04:19:28.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.057 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:28.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:28.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:28.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:28.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:28.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:28.371 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:19:28.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.464 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:28.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:28.541 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 124811 124795 109.836 76.1875 0.00759655 0.00909984 2024-09-14T04:19:28.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:28.599 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:28.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:28.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:28.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:28.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:28.885 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:28.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:28.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:29.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:29.047 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:19:29.047 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:19:29.091 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:19:29.091 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:19:29.092 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:19:29.123 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:19:29.124 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:19:29.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:29.157 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:19:29.157 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:19:29.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:29.190 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:19:29.191 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:19:29.228 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:19:29.228 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:19:29.256 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:19:29.257 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:19:29.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:29.317 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/24518883' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:29.318 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/24518883' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:29.318 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:29 smithi001 podman[263382]: 2024-09-14 04:19:29.046821379 +0000 UTC m=+0.884226506 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:19:29.318 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:29 smithi001 podman[263382]: 2024-09-14 04:19:29.205607216 +0000 UTC m=+1.043012335 container create b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T04:19:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:29.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:29.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/24518883' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:29.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:29.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:29.542 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 126058 126042 109.393 77.9375 0.00772114 0.00913687 2024-09-14T04:19:29.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:29.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:29.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:29.738 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:29.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:29.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:29.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:29.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:29.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:29.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:29 smithi001 podman[263382]: 2024-09-14 04:19:29.921987937 +0000 UTC m=+1.759393071 container init b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-14T04:19:29.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:29 smithi001 podman[263382]: 2024-09-14 04:19:29.933613812 +0000 UTC m=+1.771018935 container start b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T04:19:29.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:30.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.016 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:30.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:30.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:30.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:30.294 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:30.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:30 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 68418/364458 objects degraded (18.773%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:30.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:30 smithi012 ceph-mon[101180]: pgmap v1072: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 69205/370311 objects degraded (18.688%) 2024-09-14T04:19:30.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:30 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 68418/364458 objects degraded (18.773%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:30 smithi001 ceph-mon[112165]: pgmap v1072: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 69205/370311 objects degraded (18.688%) 2024-09-14T04:19:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:30 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 68418/364458 objects degraded (18.773%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:30.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:30 smithi001 ceph-mon[107399]: pgmap v1072: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 69205/370311 objects degraded (18.688%) 2024-09-14T04:19:30.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 127351 127335 109.001 80.8125 0.00523395 0.00916954 2024-09-14T04:19:30.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:30.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:30.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:30.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:30.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:30.837 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:30.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:30.976 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:30.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:31.077 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:31.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:31.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:31.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:31.338 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:30 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:19:31.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:19:31.379 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:19:31.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:31.496 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:31.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:31.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:31.542 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 128610 128594 108.592 78.6875 0.0130271 0.00920458 2024-09-14T04:19:31.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:31.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:31.699 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:19:31.700 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:19:31.700 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:19:31.700 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 podman[263382]: 2024-09-14 04:19:31.48819688 +0000 UTC m=+3.325602006 container attach b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T04:19:31.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.806 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:31.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:31.886 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:19:31.886 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:19:31.893 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:31.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:19:31.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate[263577]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:19:31.955 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:31 smithi001 bash[263382]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:19:31.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:31.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:32.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:32.098 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:32.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:32.220 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:32 smithi001 podman[263382]: 2024-09-14 04:19:32.182630768 +0000 UTC m=+4.020035886 container died b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-14T04:19:32.238 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:32.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:32.379 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:32.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:32.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:32.542 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 129813 129797 108.146 75.1875 0.0250653 0.00924184 2024-09-14T04:19:32.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:32.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:32.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:32 smithi001 ceph-mon[107399]: pgmap v1073: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 70707/377208 objects degraded (18.745%) 2024-09-14T04:19:32.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:32 smithi001 ceph-mon[112165]: pgmap v1073: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 70707/377208 objects degraded (18.745%) 2024-09-14T04:19:32.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:32.797 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:32 smithi012 ceph-mon[101180]: pgmap v1073: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 70707/377208 objects degraded (18.745%) 2024-09-14T04:19:32.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:32.942 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:32.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:33.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:33.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:33.169 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:33 smithi001 podman[263912]: 2024-09-14 04:19:33.058761169 +0000 UTC m=+1.091625146 container remove b978d7f77edd5d5b31b5635edbc12357ca95d034699afe529ecadcca9dc6b0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:19:33.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:33.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:33.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:33.358 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:33.380 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:19:33.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:33.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:33.542 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 131063 131047 107.751 78.125 0.0117271 0.0092765 2024-09-14T04:19:33.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:33.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:33.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:33.770 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:33.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:33.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:33.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:34.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:19:34.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:34.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:34.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:34.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:34.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:34.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:34.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 132311 132295 107.363 78 0.00700228 0.00931027 2024-09-14T04:19:34.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:34.590 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:34.633 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:34.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:34 smithi001 ceph-mon[107399]: pgmap v1074: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 71850/382983 objects degraded (18.761%) 2024-09-14T04:19:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 71850/382983 objects degraded (18.761%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:34 smithi001 ceph-mon[112165]: pgmap v1074: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 71850/382983 objects degraded (18.761%) 2024-09-14T04:19:34.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 71850/382983 objects degraded (18.761%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:34.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:34.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:34 smithi012 ceph-mon[101180]: pgmap v1074: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 71850/382983 objects degraded (18.761%) 2024-09-14T04:19:34.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 71850/382983 objects degraded (18.761%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:34.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:34.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:34.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:34.993 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:34 smithi001 podman[264173]: 2024-09-14 04:19:34.630717464 +0000 UTC m=+0.036501478 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:19:34.993 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:34 smithi001 podman[264173]: 2024-09-14 04:19:34.821546175 +0000 UTC m=+0.227330189 container create a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-14T04:19:35.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:35.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:35.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:35.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:35.279 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:35.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:35.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:35.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:35.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 133502 133486 106.941 74.4375 0.0172981 0.00934642 2024-09-14T04:19:35.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:35.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:35.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:35.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:35.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:35.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:35.830 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:35 smithi001 podman[264173]: 2024-09-14 04:19:35.510793652 +0000 UTC m=+0.916577666 container init a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-14T04:19:35.830 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:35 smithi001 podman[264173]: 2024-09-14 04:19:35.523365972 +0000 UTC m=+0.929149986 container start a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:19:35.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:35.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:35.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:36.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:36.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:36.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:36.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-osd[264307]: -- 172.21.15.1:0/1987079146 <== mon.0 v2:172.21.15.1:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5589d69cfa40 con 0x5589d69adc00 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[112165]: pgmap v1075: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 72656/388791 objects degraded (18.688%) 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:35 smithi001 ceph-mon[107399]: pgmap v1075: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 72656/388791 objects degraded (18.688%) 2024-09-14T04:19:36.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:36.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:36.329 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:35 smithi012 ceph-mon[101180]: pgmap v1075: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 72656/388791 objects degraded (18.688%) 2024-09-14T04:19:36.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:36.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:36.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:36.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:36.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 134324 134308 106.238 51.375 0.0198884 0.00940795 2024-09-14T04:19:36.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:36.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:36.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:19:36.682 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:36.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:36.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:36.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:36.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:36.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:36] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:19:36.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:36 smithi001 bash[264173]: a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 2024-09-14T04:19:36.971 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:36 smithi001 systemd[1]: Started Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:19:37.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:37.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:37.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:37.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:37.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:37.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:37.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:37.469 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:37.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:19:37.543128+0000 min lat: 0.000960706 max lat: 0.455133 avg lat: 0.00945328 2024-09-14T04:19:37.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:19:37.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 135371 135355 105.728 65.4375 0.00916065 0.00945328 2024-09-14T04:19:37.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:37.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:37.709 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:37.722 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:19:37.269+0000 7f7b44055640 -1 Falling back to public interface 2024-09-14T04:19:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.754 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:37.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:37.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:37.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:37.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:37.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:38.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:38.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:38.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:38.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:38.320 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:38.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:38.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:38.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 136584 136568 105.358 75.8125 0.0337739 0.00948699 2024-09-14T04:19:38.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:38.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:38.677 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:38 smithi001 ceph-mon[107399]: pgmap v1076: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 75183/400782 objects degraded (18.759%) 2024-09-14T04:19:38.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:38 smithi001 ceph-mon[112165]: pgmap v1076: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 75183/400782 objects degraded (18.759%) 2024-09-14T04:19:38.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:38.746 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:38 smithi012 ceph-mon[101180]: pgmap v1076: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 75183/400782 objects degraded (18.759%) 2024-09-14T04:19:38.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:38.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:38.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:38.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:39.016 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:39.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:39.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:39.161 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:39.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:39.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:39.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:39.431 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:19:39.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:39.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:39.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 137778 137762 104.984 74.625 0.0179289 0.00952089 2024-09-14T04:19:39.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:39.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3924065356' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 75183/400782 objects degraded (18.759%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:39.633 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3924065356' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.633 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 75183/400782 objects degraded (18.759%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:39.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:39.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:39.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3924065356' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:19:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 75183/400782 objects degraded (18.759%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:39.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:39.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:39.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.028 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:40.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:40.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:40.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:40.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:40.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:40.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:40.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:40.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 138954 138938 104.604 73.5 0.0133848 0.00955569 2024-09-14T04:19:40.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:40.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:40.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:40.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[112165]: pgmap v1077: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 75885/405948 objects degraded (18.693%) 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[112165]: osdmap e295: 8 total, 7 up, 8 in 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[107399]: pgmap v1077: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 75885/405948 objects degraded (18.693%) 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:40 smithi001 ceph-mon[107399]: osdmap e295: 8 total, 7 up, 8 in 2024-09-14T04:19:40.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:40 smithi012 ceph-mon[101180]: pgmap v1077: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 75885/405948 objects degraded (18.693%) 2024-09-14T04:19:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:40 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:19:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:40 smithi012 ceph-mon[101180]: osdmap e295: 8 total, 7 up, 8 in 2024-09-14T04:19:40.848 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:40.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:40.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:40.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:40.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:41.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:41.125 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:41.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:41.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:41.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:41.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:41.406 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:41.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:41.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:41.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 140090 140074 104.204 71 0.0173759 0.00959163 2024-09-14T04:19:41.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:41.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:41.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:41.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:41.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:41.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:41.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:41.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:42.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:42.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:42.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:42.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:42.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:42.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:42 smithi001 ceph-mon[112165]: pgmap v1079: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 76626/409386 objects degraded (18.717%) 2024-09-14T04:19:42.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:42 smithi001 ceph-mon[107399]: pgmap v1079: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 76626/409386 objects degraded (18.717%) 2024-09-14T04:19:42.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:42.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:42.543 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 141188 141172 103.785 68.625 0.0155221 0.00963103 2024-09-14T04:19:42.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:42.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:42.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:42 smithi012 ceph-mon[101180]: pgmap v1079: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 76626/409386 objects degraded (18.717%) 2024-09-14T04:19:42.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:42.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:42.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:43.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:43.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:43.127 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:43.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:43.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:43.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:43.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:43.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.506 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:43.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:43.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 142286 142270 103.377 68.625 0.00882716 0.0096686 2024-09-14T04:19:43.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:43.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:43.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:43.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:43.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:43.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:43.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:44.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:44.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:44.151 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:44.179 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:44.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:44.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:44.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:44.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:44.526 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:44 smithi001 ceph-mon[107399]: pgmap v1080: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.5 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 78367/417888 objects degraded (18.753%) 2024-09-14T04:19:44.526 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 78367/417888 objects degraded (18.753%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:44.526 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:44 smithi001 ceph-mon[112165]: pgmap v1080: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.5 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 78367/417888 objects degraded (18.753%) 2024-09-14T04:19:44.526 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 78367/417888 objects degraded (18.753%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:44.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 143475 143459 103.042 74.3125 0.0110837 0.00969986 2024-09-14T04:19:44.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:44.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:44.658 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:19:44.695 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:44.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:44.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:44 smithi012 ceph-mon[101180]: pgmap v1080: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.5 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 78367/417888 objects degraded (18.753%) 2024-09-14T04:19:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 78367/417888 objects degraded (18.753%), 12 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:44.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:44.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:44.864 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:19:44.864 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:19:44.915 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:19:44.916 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:19:44.924 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-09-14T04:19:44.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:44.963 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:19:44.963 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:19:44.967 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:19:44.967 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:19:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.001 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:19:45.001 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:19:45.002 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:45.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:45.034 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:19:45.034 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:19:45.062 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:19:45.062 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:19:45.099 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:19:45.100 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:19:45.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:45.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:45.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:45.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:45.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:45.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:45.428 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:19:45.272+0000 7f7b44055640 -1 osd.2 259 log_to_monitors true 2024-09-14T04:19:45.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:45.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 144541 144525 102.629 66.625 0.0138253 0.00974008 2024-09-14T04:19:45.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:45.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:45.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.701 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:45.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1273260782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[112165]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[112165]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1273260782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[107399]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-mon[107399]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:45.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:45.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1273260782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:45 smithi012 ceph-mon[101180]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:45 smithi012 ceph-mon[101180]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:19:45.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:45.988 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:45.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:46.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:46.128 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:46.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:46.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:19:45 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:19:45.739+0000 7f7b3bdff640 -1 osd.2 296 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:19:46.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:46.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:46.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:46.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:46.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:46.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:46.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 145657 145641 102.259 69.75 0.00604205 0.00976515 2024-09-14T04:19:46.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:46.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:46.701 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[107399]: pgmap v1081: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 79062/423195 objects degraded (18.682%) 2024-09-14T04:19:46.701 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[107399]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[107399]: osdmap e296: 8 total, 7 up, 8 in 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[107399]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[107399]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[112165]: pgmap v1081: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 79062/423195 objects degraded (18.682%) 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[112165]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:19:46.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[112165]: osdmap e296: 8 total, 7 up, 8 in 2024-09-14T04:19:46.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[112165]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-mon[112165]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:46.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:46 smithi012 ceph-mon[101180]: pgmap v1081: 109 pgs: 3 active+clean+remapped, 19 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 79062/423195 objects degraded (18.682%) 2024-09-14T04:19:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:46 smithi012 ceph-mon[101180]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:19:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:46 smithi012 ceph-mon[101180]: osdmap e296: 8 total, 7 up, 8 in 2024-09-14T04:19:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:46 smithi012 ceph-mon[101180]: from='osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:46 smithi012 ceph-mon[101180]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi001", "root=default"]} : dispatch 2024-09-14T04:19:46.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:46.899 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:46.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:46] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-09-14T04:19:47.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:47.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:19:47.038 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:19:47.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:47.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:47.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:47.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:47.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:47.397 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:47.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:47.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:19:47.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:47.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:47.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 90 16 145657 145641 101.123 0 - 0.00976515 2024-09-14T04:19:47.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:47.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:19:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[107399]: osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503] boot 2024-09-14T04:19:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[107399]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T04:19:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:19:47.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[107399]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T04:19:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:19:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[112165]: osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503] boot 2024-09-14T04:19:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[112165]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T04:19:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:19:47.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:47 smithi001 ceph-mon[112165]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T04:19:47.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:47.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:47 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:19:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:47 smithi012 ceph-mon[101180]: osd.2 [v2:172.21.15.1:6818/1732438503,v1:172.21.15.1:6819/1732438503] boot 2024-09-14T04:19:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:47 smithi012 ceph-mon[101180]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T04:19:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:19:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:47 smithi012 ceph-mon[101180]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T04:19:47.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:47.905 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:47.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:47.966 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:19:47.967 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:19:47.967 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:19:47.967 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:19:48.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:48.033 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:48.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:48.161 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:48.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:48.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:48.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:48.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:48.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:48.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 91 16 145657 145641 100.012 0 - 0.00976515 2024-09-14T04:19:48.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:48.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:48 smithi001 ceph-mon[107399]: pgmap v1084: 109 pgs: 13 peering, 3 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 75 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 53921/434265 objects degraded (12.417%) 2024-09-14T04:19:48.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:48 smithi001 ceph-mon[107399]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T04:19:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:48 smithi001 ceph-mon[112165]: pgmap v1084: 109 pgs: 13 peering, 3 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 75 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 53921/434265 objects degraded (12.417%) 2024-09-14T04:19:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:48 smithi001 ceph-mon[112165]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T04:19:48.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:48.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:48 smithi012 ceph-mon[101180]: pgmap v1084: 109 pgs: 13 peering, 3 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 75 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 53921/434265 objects degraded (12.417%) 2024-09-14T04:19:48.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:48 smithi012 ceph-mon[101180]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T04:19:48.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:48.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:48.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:49.022 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:49.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:19:49.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:49.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:49.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:49.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:49.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:49.322 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:49.359 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:49.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:49.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:49.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:49.544 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 91.6036 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 145657 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 99.3799 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 44.6793 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 265.125 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1590 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 714.869 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 4242 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0100416 2024-09-14T04:19:49.545 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.0279648 2024-09-14T04:19:49.546 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 2.69744 2024-09-14T04:19:49.546 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.000960706 2024-09-14T04:19:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:49.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:49.551 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:19:49.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:49.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:49.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.a starting backfill to osd.2 from (0'0,0'0] MAX to 296'9279 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.c starting backfill to osd.2 from (0'0,0'0] MAX to 296'9125 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.b starting backfill to osd.2 from (0'0,0'0] MAX to 296'9089 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.8 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8788 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.d starting backfill to osd.2 from (0'0,0'0] MAX to 296'9177 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.7 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9192 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.9 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8989 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: 12.0 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9046 2024-09-14T04:19:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 53921/434265 objects degraded (12.417%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.a starting backfill to osd.2 from (0'0,0'0] MAX to 296'9279 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.c starting backfill to osd.2 from (0'0,0'0] MAX to 296'9125 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.b starting backfill to osd.2 from (0'0,0'0] MAX to 296'9089 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.8 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8788 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.d starting backfill to osd.2 from (0'0,0'0] MAX to 296'9177 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.7 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9192 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.9 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8989 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: 12.0 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9046 2024-09-14T04:19:49.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 53921/434265 objects degraded (12.417%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:49.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:49.820 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.a starting backfill to osd.2 from (0'0,0'0] MAX to 296'9279 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.c starting backfill to osd.2 from (0'0,0'0] MAX to 296'9125 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.b starting backfill to osd.2 from (0'0,0'0] MAX to 296'9089 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.8 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8788 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.d starting backfill to osd.2 from (0'0,0'0] MAX to 296'9177 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.7 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9192 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.9 starting backfill to osd.2 from (0'0,0'0] MAX to 296'8989 2024-09-14T04:19:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: 12.0 starting backfill to osd.2 from (0'0,0'0] MAX to 296'9046 2024-09-14T04:19:49.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 53921/434265 objects degraded (12.417%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:49.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:49.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:49.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:50.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:50.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:50.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:50.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:50.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:50.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:50.385 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:50.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:50.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:50.536 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:50.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:50.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[112165]: pgmap v1087: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 80 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 54278/436833 objects degraded (12.425%) 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[107399]: pgmap v1087: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 80 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 54278/436833 objects degraded (12.425%) 2024-09-14T04:19:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:50.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:50.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:50.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:50.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:50 smithi012 ceph-mon[101180]: pgmap v1087: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 80 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 54278/436833 objects degraded (12.425%) 2024-09-14T04:19:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:50.862 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:50.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:50.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:50.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:50.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:51.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:51.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:51.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:51.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:51.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:51.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:51.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:51.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:51.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:51.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:51.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:51.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:51.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:51.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:51.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:51.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:51.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:51.953 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:51.953 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:51.954 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[112165]: pgmap v1088: 109 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+undersized+degraded, 90 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 48 MiB/s wr, 812 op/s; 54016/436263 objects degraded (12.382%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:19:51.954 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:51.954 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:51.954 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:51 smithi001 ceph-mon[107399]: pgmap v1088: 109 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+undersized+degraded, 90 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 48 MiB/s wr, 812 op/s; 54016/436263 objects degraded (12.382%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:19:52.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:52.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:51 smithi012 ceph-mon[101180]: pgmap v1088: 109 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 13 peering, 2 active+undersized+degraded, 90 active+clean; 8.9 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 48 MiB/s wr, 812 op/s; 54016/436263 objects degraded (12.382%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:19:52.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:52.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:52.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:52.326 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:52.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:52.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:52.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:52.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:52.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:52.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:52.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:52.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:52.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:52.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:52.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:52.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:53.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:53.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:53.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:53.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:53.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:53.372 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2058531871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:53.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:53.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:53.448 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2058531871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:53.448 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2058531871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:19:53.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:53.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:53.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:53.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:53.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:53.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:53.880 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:53.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:53.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:54.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:54.109 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:19:54.109 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:19:54.109 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:19:54.110 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:19:54.111 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:19:54.111 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:19:54.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:54.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:54.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:54.294 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2024-09-14T04:19:54.295 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:19:54.295 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-14T04:19:54.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-14T04:19:54.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:54.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:54.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:54.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:54 smithi012 ceph-mon[101180]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 147 B/s rd, 23 MiB/s wr, 536 op/s; 81025/433410 objects degraded (18.695%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:19:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 54016/436263 objects degraded (12.382%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:54.652 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:54 smithi001 ceph-mon[107399]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 147 B/s rd, 23 MiB/s wr, 536 op/s; 81025/433410 objects degraded (18.695%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:19:54.653 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 54016/436263 objects degraded (12.382%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:54.653 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:54 smithi001 ceph-mon[112165]: pgmap v1089: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.8 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 147 B/s rd, 23 MiB/s wr, 536 op/s; 81025/433410 objects degraded (18.695%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:19:54.653 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 54016/436263 objects degraded (12.382%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:19:54.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:54.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:54.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:54.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:54.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:54.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:55.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:55.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:55.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:55.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:55.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:55.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:55.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.415 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:55.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:55.421 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:19:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=cleanup t=2024-09-14T04:19:55.274487489Z level=info msg="Completed cleanup jobs" duration=24.622816ms 2024-09-14T04:19:55.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:55.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:55.652 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:55.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:55.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:55.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:55.846 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:19:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugins.update.checker t=2024-09-14T04:19:55.421303068Z level=info msg="Update check succeeded" duration=62.035891ms 2024-09-14T04:19:55.846 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:19:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana.update.checker t=2024-09-14T04:19:55.429818298Z level=info msg="Update check succeeded" duration=61.73867ms 2024-09-14T04:19:55.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:55.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:55.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:56.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:56.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:56.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:56.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:56.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.396 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:56.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:56.420 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:19:56.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:56.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:56.631 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:56.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[112165]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 7.2 MiB/s wr, 420 op/s; 80540/429741 objects degraded (18.742%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[107399]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 7.2 MiB/s wr, 420 op/s; 80540/429741 objects degraded (18.742%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:19:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:56.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:56.803 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:56.807 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:56 smithi012 ceph-mon[101180]: pgmap v1090: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 7.2 MiB/s wr, 420 op/s; 80540/429741 objects degraded (18.742%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:19:56.807 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:19:56.807 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:56 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:19:56.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:56.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:56.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:19:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:19:56] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.51.0" 2024-09-14T04:19:57.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:19:57.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:19:57.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:57.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:19:57.172 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:57.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:57.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:57.339 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:19:57.340 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:19:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.366 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:57.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:57.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:57.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:57.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:57.638 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:57.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[107399]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[112165]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:57 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:57.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/730430896' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:19:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:57 smithi012 ceph-mon[101180]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T04:19:57.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:57 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:57 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:19:57.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:57.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:57.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:58.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:58.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:19:58.144 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:19:58.186 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:19:58.288 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:58.323 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:58.425 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:19:58.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:19:58.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:58.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:19:58.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:58.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:58.642 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:58.712 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:19:58.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[107399]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 114 B/s rd, 6.5 MiB/s wr, 718 op/s; 78895/420597 objects degraded (18.758%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:19:58.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[107399]: 1.11 scrub starts 2024-09-14T04:19:58.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[107399]: 1.11 scrub ok 2024-09-14T04:19:58.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[112165]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 114 B/s rd, 6.5 MiB/s wr, 718 op/s; 78895/420597 objects degraded (18.758%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:19:58.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[112165]: 1.11 scrub starts 2024-09-14T04:19:58.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:58 smithi001 ceph-mon[112165]: 1.11 scrub ok 2024-09-14T04:19:58.770 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:19:58.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:19:58.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:19:58.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:58 smithi012 ceph-mon[101180]: pgmap v1092: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 114 B/s rd, 6.5 MiB/s wr, 718 op/s; 78895/420597 objects degraded (18.758%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:19:58.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:58 smithi012 ceph-mon[101180]: 1.11 scrub starts 2024-09-14T04:19:58.848 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:58 smithi012 ceph-mon[101180]: 1.11 scrub ok 2024-09-14T04:19:58.866 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:19:58.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:58.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:58.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:59.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:59.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:59.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:59.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:19:59.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:59.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:59.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:19:59.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:19:59.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:19:59.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:19:59.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:59.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:19:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:19:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:19:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:19:59.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:19:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:19:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:19:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:19:59.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:19:59.979 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:19:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:19:59.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:00.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:00.118 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:00.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.265 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:00.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:00.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:00.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:00.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:00.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:00.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:00.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:00.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pgmap v1093: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 461 KiB/s wr, 692 op/s; 78352/416505 objects degraded (18.812%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: [WRN] PG_DEGRADED: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.0 is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.5 is active+recovering+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,4] 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.8 is active+undersized+degraded+remapped+backfill_wait, acting [3,0] 2024-09-14T04:20:00.721 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.9 is active+undersized+degraded+remapped+backfill_wait, acting [1,3] 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.a is active+undersized+degraded+remapped+backfill_wait, acting [0,3] 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.b is active+undersized+degraded+remapped+backfill_wait, acting [3,4] 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.c is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:00 smithi012 ceph-mon[101180]: pg 12.d is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pgmap v1093: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 461 KiB/s wr, 692 op/s; 78352/416505 objects degraded (18.812%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: [WRN] PG_DEGRADED: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.0 is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.5 is active+recovering+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,4] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.8 is active+undersized+degraded+remapped+backfill_wait, acting [3,0] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.9 is active+undersized+degraded+remapped+backfill_wait, acting [1,3] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.a is active+undersized+degraded+remapped+backfill_wait, acting [0,3] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.b is active+undersized+degraded+remapped+backfill_wait, acting [3,4] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.c is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[112165]: pg 12.d is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-09-14T04:20:00.723 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pgmap v1093: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 461 KiB/s wr, 692 op/s; 78352/416505 objects degraded (18.812%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: [WRN] PG_DEGRADED: Degraded data redundancy: 78895/420597 objects degraded (18.758%), 9 pgs degraded 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.0 is active+undersized+degraded+remapped+backfill_wait, acting [4,7] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.5 is active+recovering+undersized+degraded+remapped, acting [6,1] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,4] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.8 is active+undersized+degraded+remapped+backfill_wait, acting [3,0] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.9 is active+undersized+degraded+remapped+backfill_wait, acting [1,3] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.a is active+undersized+degraded+remapped+backfill_wait, acting [0,3] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.b is active+undersized+degraded+remapped+backfill_wait, acting [3,4] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.c is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2024-09-14T04:20:00.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:00 smithi001 ceph-mon[107399]: pg 12.d is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-09-14T04:20:00.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.810 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:00.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:00.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:00.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:00.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:01.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:01.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:01.136 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:01.137 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:01.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:01.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:01.313 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:01.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:01.422 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:01.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:01.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:01.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:01.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:01.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:01.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:01.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:01.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:20:01.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:01.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:01.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:02.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:02.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:02.138 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:02.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:02.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:02.262 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:20:02.262 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:20:02.286 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:20:02.287 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:20:02.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:02.317 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:20:02.317 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:20:02.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:02.347 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:20:02.347 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:20:02.379 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:20:02.379 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:20:02.407 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:20:02.407 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:20:02.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:02.439 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:20:02.440 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:20:02.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:02.473 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:20:02.473 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:20:02.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:02 smithi012 ceph-mon[101180]: pgmap v1094: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 198 KiB/s wr, 752 op/s; 77763/413862 objects degraded (18.790%); 621 KiB/s, 9 objects/s recovering 2024-09-14T04:20:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3784046652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.597 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:02.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:02.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:02 smithi001 ceph-mon[107399]: pgmap v1094: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 198 KiB/s wr, 752 op/s; 77763/413862 objects degraded (18.790%); 621 KiB/s, 9 objects/s recovering 2024-09-14T04:20:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3784046652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:02 smithi001 ceph-mon[112165]: pgmap v1094: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 198 KiB/s wr, 752 op/s; 77763/413862 objects degraded (18.790%); 621 KiB/s, 9 objects/s recovering 2024-09-14T04:20:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3784046652' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:02.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:02.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:02.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:02.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:02.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:02.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:02.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:03.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:03.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.269 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:03.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:03.399 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:03.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:03.536 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:03.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:03.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:03.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:03.806 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:03.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:03.938 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:03.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:04.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:04.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:04.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:04.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:04.309 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:04.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:04.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:04.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:04.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:04.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:04.626 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:04 smithi001 ceph-mon[107399]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 76550/407043 objects degraded (18.806%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 76550/407043 objects degraded (18.806%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:04 smithi001 ceph-mon[112165]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 76550/407043 objects degraded (18.806%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 76550/407043 objects degraded (18.806%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:04.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:04.742 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:04 smithi012 ceph-mon[101180]: pgmap v1095: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.3 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 76550/407043 objects degraded (18.806%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:04.743 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 76550/407043 objects degraded (18.806%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:04.767 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:04.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:04.907 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:04.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:05.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:05.039 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:05.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:05.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:05.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:05.320 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:05.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:05.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:05.465 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:20:05.466 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:20:05.466 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:20:05.466 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:20:05.466 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:20:05.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:05.592 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:05.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-14T04:20:05.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:05.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:05.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:05.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:05.875 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:05.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:05.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:05.993 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:20:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:06.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:06.154 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:06.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:06.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:20:06.256 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:06.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:06.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:06.433 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:06.532 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:06.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:06.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:06.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:06.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[107399]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 75980/402801 objects degraded (18.863%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[107399]: 1.1 scrub starts 2024-09-14T04:20:06.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[107399]: 1.1 scrub ok 2024-09-14T04:20:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[112165]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 75980/402801 objects degraded (18.863%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[112165]: 1.1 scrub starts 2024-09-14T04:20:06.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-mon[112165]: 1.1 scrub ok 2024-09-14T04:20:06.722 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:06] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.51.0" 2024-09-14T04:20:06.809 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:06 smithi012 ceph-mon[101180]: pgmap v1096: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.2 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 75980/402801 objects degraded (18.863%); 595 KiB/s, 9 objects/s recovering 2024-09-14T04:20:06.809 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:06 smithi012 ceph-mon[101180]: 1.1 scrub starts 2024-09-14T04:20:06.809 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:06 smithi012 ceph-mon[101180]: 1.1 scrub ok 2024-09-14T04:20:06.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:06.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:06.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:06.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:06.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:07.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:07.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:07.238 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:07.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:07.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:07.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:07.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:07.615 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:07.624 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:07.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:07.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:07.767 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:07.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:07.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:07.910 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:07.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:08.012 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:08.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:08.149 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:08.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:08.290 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:08.325 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:08.427 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:08.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:08.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.465 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:161525 10.1 10:8b2d5aa2:::benchmark_data_smithi001_190881_object46385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 516.35890964299995, 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.191356781, 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 161525 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:20:08.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102700+0000", 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102698+0000", 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102705+0000", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0960000000000003e-06 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102720+0000", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4832e-05 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102727+0000", 2024-09-14T04:20:08.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3819999999999998e-06 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102789+0000", 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1450000000000003e-05 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.102903+0000", 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011477899999999999 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.103016+0000", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112229 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.109259+0000", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062428309999999999 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.111276+0000", 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020170819999999999 2024-09-14T04:20:08.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.293948+0000", 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.182672263 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.294010+0000", 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2498999999999999e-05 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:11:32.294057+0000", 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6244e-05 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:190078 10.e 10:76b6074a:::benchmark_data_smithi001_190881_object74938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 477.81110377599998, 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28708630899999998, 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190078 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650506+0000", 2024-09-14T04:20:08.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650504+0000", 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650513+0000", 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5440000000000004e-06 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650534+0000", 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1009000000000002e-05 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650543+0000", 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1910000000000006e-06 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650591+0000", 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7670000000000003e-05 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650681+0000", 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9757000000000001e-05 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.650768+0000", 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6849999999999997e-05 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.654736+0000", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039679140000000003 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.655049+0000", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031341799999999999 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937482+0000", 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.282432823 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937542+0000", 2024-09-14T04:20:08.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0265e-05 2024-09-14T04:20:08.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.937592+0000", 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0016999999999999e-05 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:190089 10.e 10:7077390a:::benchmark_data_smithi001_190881_object74949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 477.78873254400003, 2024-09-14T04:20:08.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26623560299999999, 2024-09-14T04:20:08.480 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190089 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.481 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.483 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:20:08.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672877+0000", 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672876+0000", 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672883+0000", 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672896+0000", 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3298e-05 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672904+0000", 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9279999999999993e-06 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.672960+0000", 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6360000000000002e-05 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.673104+0000", 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014360499999999999 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.673206+0000", 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102457 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.676581+0000", 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033750500000000001 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.676693+0000", 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011162000000000001 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939040+0000", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26234669300000002 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939074+0000", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3766999999999997e-05 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:10.939113+0000", 2024-09-14T04:20:08.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8986000000000002e-05 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:212503 10.e 10:77b7a991:::benchmark_data_smithi001_190881_object97363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 448.49401242599998, 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.138707203, 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.490 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212503 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967597+0000", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967595+0000", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967603+0000", 2024-09-14T04:20:08.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9559999999999997e-06 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967617+0000", 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3786e-05 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967623+0000", 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5209999999999999e-06 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967662+0000", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9100000000000002e-05 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967775+0000", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112329 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.967902+0000", 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127465 2024-09-14T04:20:08.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.972344+0000", 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044414340000000002 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.973465+0000", 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001121274 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106206+0000", 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13274113500000001 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106260+0000", 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3634000000000002e-05 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.106304+0000", 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4678000000000002e-05 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.496 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45035.0:212509 10.e 10:796e96a7:::benchmark_data_smithi001_190881_object97369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 448.48267094099998, 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.131838491, 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.497 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45035", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3969366944", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 212509 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978939+0000", 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978937+0000", 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978946+0000", 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0299999999999999e-06 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978961+0000", 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.518e-05 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.978968+0000", 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3320000000000003e-06 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979020+0000", 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1502000000000003e-05 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979235+0000", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021467200000000001 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.979370+0000", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013521900000000001 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.985494+0000", 2024-09-14T04:20:08.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006124394 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:39.987278+0000", 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001783972 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110683+0000", 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.123405265 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110736+0000", 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2460999999999998e-05 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:12:40.110777+0000", 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1102000000000001e-05 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.503 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111745 11.d 11:b3f626a6:::benchmark_data_smithi001_228075_object111744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 276.03177716699997, 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0046928020000001, 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111745 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429832+0000", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429831+0000", 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429860+0000", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9505e-05 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429872+0000", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1993e-05 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429880+0000", 2024-09-14T04:20:08.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5349999999999999e-06 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429939+0000", 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9515999999999997e-05 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.429947+0000", 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0739999999999993e-06 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.432067+0000", 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0021197639999999 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.434525+0000", 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024579860000000001 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111755 11.d 11:b0ede1e3:::benchmark_data_smithi001_228075_object111754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:20:08.508 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 276.02644288300002, 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.006606546, 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111755 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435167+0000", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435165+0000", 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435200+0000", 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5256999999999998e-05 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435222+0000", 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2025999999999999e-05 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435232+0000", 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6900000000000004e-06 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435282+0000", 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0390999999999999e-05 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435291+0000", 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1880000000000008e-06 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.434878+0000", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99958683299999995 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.435747+0000", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00086862599999999995 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.435903+0000", 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015575100000000001 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440083+0000", 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041805849999999997 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441626+0000", 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015432309999999999 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441693+0000", 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7076999999999995e-05 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441773+0000", 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9715000000000005e-05 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117532 11.d 11:ba26fc87:::benchmark_data_smithi001_228075_object117531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.56350570199999, 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13377942700000001, 2024-09-14T04:20:08.516 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117532 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898104+0000", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898103+0000", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898166+0000", 2024-09-14T04:20:08.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3083000000000006e-05 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898176+0000", 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0638999999999999e-05 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.898184+0000", 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0450000000000003e-06 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902114+0000", 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039306489999999996 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902263+0000", 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014865000000000001 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902410+0000", 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147465 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.021140+0000", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.118729843 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031763+0000", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010622509 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031809+0000", 2024-09-14T04:20:08.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6316000000000002e-05 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031883+0000", 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4270999999999994e-05 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117538 11.d 11:b56e9153:::benchmark_data_smithi001_228075_object117537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:20:08.522 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.558856053, 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13954377000000001, 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117538 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902753+0000", 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902752+0000", 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.902792+0000", 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0565999999999998e-05 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.013440+0000", 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11064774600000001 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.013457+0000", 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6597999999999999e-05 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.021164+0000", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0077068120000000004 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.022584+0000", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014198679999999999 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.022732+0000", 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014866 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.031891+0000", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091587700000000001 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.041218+0000", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093267009999999997 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.041275+0000", 2024-09-14T04:20:08.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7840000000000002e-05 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:40.042297+0000", 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010217340000000001 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:69528 12.2 12:4975f13a:::benchmark_data_smithi001_255951_object69527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:18:46.641971+0000", 2024-09-14T04:20:08.528 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 81.819638847999997, 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14865016, 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 69528 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.641971+0000", 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.641971+0000", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.641969+0000", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642060+0000", 2024-09-14T04:20:08.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0805999999999998e-05 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642080+0000", 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9700000000000001e-05 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642094+0000", 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4749999999999999e-05 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642164+0000", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9212000000000001e-05 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642314+0000", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015029799999999999 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.642409+0000", 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5336999999999999e-05 2024-09-14T04:20:08.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.654570+0000", 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012160822999999999 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656763+0000", 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021931479999999998 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790529+0000", 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.133765989 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790581+0000", 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2157999999999998e-05 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790621+0000", 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9496000000000003e-05 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:69541 12.2 12:4b73f21b:::benchmark_data_smithi001_255951_object69540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:18:46.649432+0000", 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 81.812177207000005, 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14129788300000001, 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 69541 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.649432+0000", 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.649432+0000", 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.649431+0000", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.649506+0000", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5069999999999998e-05 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.652325+0000", 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028191919999999999 2024-09-14T04:20:08.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.652336+0000", 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1656e-05 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.654597+0000", 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022602590000000001 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.654729+0000", 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013271800000000001 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.654857+0000", 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012768499999999999 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.659224+0000", 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043666130000000001 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.659244+0000", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0352000000000001e-05 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790654+0000", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13140983000000001 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790700+0000", 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6057000000000003e-05 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790730+0000", 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0238000000000001e-05 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:69550 12.2 12:421ab85b:::benchmark_data_smithi001_255951_object69549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:18:46.656521+0000", 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 81.805088601999998, 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.134302271, 2024-09-14T04:20:08.541 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 69550 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656521+0000", 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656521+0000", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656520+0000", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656553+0000", 2024-09-14T04:20:08.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3045e-05 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656564+0000", 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1435e-05 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656569+0000", 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7509999999999999e-06 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656780+0000", 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000210993 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.656913+0000", 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013326599999999999 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.657004+0000", 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1174999999999994e-05 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.659333+0000", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023290160000000002 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.659581+0000", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000247156 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790760+0000", 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13117978399999999 2024-09-14T04:20:08.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790794+0000", 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3979000000000001e-05 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:18:46.790823+0000", 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8829000000000001e-05 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:106550 12.b 12:d3cc14ed:::benchmark_data_smithi001_255951_object106549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:13.895295+0000", 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 54.566314020999997, 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045635307999999999, 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 106550 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.548 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895295+0000", 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895295+0000", 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895295+0000", 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895345+0000", 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0436999999999998e-05 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895357+0000", 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.225e-05 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.895364+0000", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2330000000000003e-06 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.899219+0000", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038552399999999998 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.899362+0000", 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000143299 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.899490+0000", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012822800000000001 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.902579+0000", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030883600000000001 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.940781+0000", 2024-09-14T04:20:08.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038202528999999999 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.940859+0000", 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7822000000000001e-05 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:13.940931+0000", 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1774999999999996e-05 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:110765 12.b 12:d0dc9253:::benchmark_data_smithi001_255951_object110764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:17.314108+0000", 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 51.147501376999998, 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064542978000000001, 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 110765 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314108+0000", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314108+0000", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314107+0000", 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:08.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314161+0000", 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4242e-05 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314171+0000", 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0107e-05 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314176+0000", 2024-09-14T04:20:08.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9259999999999999e-06 2024-09-14T04:20:08.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314214+0000", 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7697000000000002e-05 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314354+0000", 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013988600000000001 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3", 2024-09-14T04:20:08.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.314493+0000", 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013904999999999999 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.317126+0000", 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026327170000000001 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.378470+0000", 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061344605000000003 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.378568+0000", 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7295999999999998e-05 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:17.378651+0000", 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3500999999999999e-05 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.560 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:122618 12.2 12:44fe58de:::benchmark_data_smithi001_255951_object122617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:26.774082+0000", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.687527445999997, 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049950725000000001, 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122618 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774082+0000", 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774082+0000", 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774081+0000", 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774143+0000", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2349999999999998e-05 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774159+0000", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.506e-05 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774166+0000", 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7060000000000004e-06 2024-09-14T04:20:08.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.783287+0000", 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091205809999999991 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.783419+0000", 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013170400000000001 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.783565+0000", 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014599099999999999 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.787924+0000", 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004359735 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.797752+0000", 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098275250000000001 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.823950+0000", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0261979 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.823997+0000", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7157999999999998e-05 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.824033+0000", 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5966000000000001e-05 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.566 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:122619 12.2 12:463c3dfa:::benchmark_data_smithi001_255951_object122618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:26.774178+0000", 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.687431162999999, 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049956519999999997, 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122619 2024-09-14T04:20:08.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774178+0000", 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774178+0000", 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774178+0000", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774241+0000", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3108999999999995e-05 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774251+0000", 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8959999999999998e-06 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.774257+0000", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.528e-06 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.787949+0000", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013692179000000001 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.788092+0000", 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014264900000000001 2024-09-14T04:20:08.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.788816+0000", 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00072409999999999998 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.797759+0000", 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089424770000000008 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.797788+0000", 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9416000000000001e-05 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.824071+0000", 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026282851999999999 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.824106+0000", 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5346999999999999e-05 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:26.824135+0000", 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8640999999999999e-05 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:124111 12.2 12:472fbcbf:::benchmark_data_smithi001_255951_object124110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:27.941164+0000", 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 40.520445541999997, 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062165870999999998, 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.573 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124111 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941164+0000", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941164+0000", 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941162+0000", 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941223+0000", 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1353999999999996e-05 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941236+0000", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2096000000000001e-05 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941242+0000", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6950000000000003e-06 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941289+0000", 2024-09-14T04:20:08.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6523999999999997e-05 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941431+0000", 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141908 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.941564+0000", 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133044 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.945622+0000", 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040578230000000003 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.945644+0000", 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2212e-05 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.003184+0000", 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057539990999999999 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.003284+0000", 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100437 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.003330+0000", 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5627000000000002e-05 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.579 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:124130 12.2 12:46fdf457:::benchmark_data_smithi001_255951_object124129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:27.958363+0000", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 40.503246193000003, 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045808569, 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124130 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958363+0000", 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958363+0000", 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958361+0000", 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958410+0000", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9425000000000002e-05 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958422+0000", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2089999999999999e-05 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958428+0000", 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9970000000000004e-06 2024-09-14T04:20:08.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958478+0000", 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9870999999999997e-05 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958579+0000", 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100715 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:20:08.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.958724+0000", 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014473399999999999 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.961559+0000", 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028351090000000002 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:27.961956+0000", 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039731100000000002 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.003369+0000", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041412956000000001 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.004138+0000", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076914300000000002 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:28.004172+0000", 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3698000000000002e-05 2024-09-14T04:20:08.585 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145650 12.0 12:0d88ba51:::benchmark_data_smithi001_255951_object145649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.461592+0000", 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 22.000017678999999, 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5042304290000001, 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.586 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145650 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461592+0000", 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461592+0000", 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461590+0000", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461652+0000", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2008999999999995e-05 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461666+0000", 2024-09-14T04:20:08.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4449e-05 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461683+0000", 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6966e-05 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461740+0000", 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6653000000000001e-05 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.461748+0000", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.447e-06 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.441555+0000", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97980794599999999 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.441569+0000", 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3515e-05 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.959465+0000", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5178957020000001 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.959684+0000", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219001 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.959886+0000", 2024-09-14T04:20:08.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000202764 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965660+0000", 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057732670000000003 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965712+0000", 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2608999999999997e-05 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965719+0000", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.319e-06 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965769+0000", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0489999999999999e-05 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965822+0000", 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3102000000000001e-05 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.593 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145655 12.0 12:02a81fc4:::benchmark_data_smithi001_255951_object145654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.470409+0000", 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 21.991200887000002, 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.496124848, 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145655 2024-09-14T04:20:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470409+0000", 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470409+0000", 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:08.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470407+0000", 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470458+0000", 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0451000000000002e-05 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470471+0000", 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3750999999999999e-05 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470477+0000", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7980000000000002e-06 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470527+0000", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0037000000000002e-05 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.470537+0000", 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0295000000000001e-05 2024-09-14T04:20:08.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.441618+0000", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97108062100000003 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.441628+0000", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8810000000000008e-06 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.962951+0000", 2024-09-14T04:20:08.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5213226500000001 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.963065+0000", 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114629 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.963185+0000", 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120122 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965984+0000", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027985029999999999 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.965989+0000", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9409999999999998e-06 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.966369+0000", 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038014800000000002 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.966426+0000", 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7562999999999998e-05 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.966533+0000", 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107019 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:08.601 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:20:08.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:08.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:08.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[107399]: pgmap v1097: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 74269/393645 objects degraded (18.867%); 790 KiB/s, 12 objects/s recovering 2024-09-14T04:20:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/466211098' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[112165]: pgmap v1097: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 74269/393645 objects degraded (18.867%); 790 KiB/s, 12 objects/s recovering 2024-09-14T04:20:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/466211098' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:08 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:08.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:08 smithi012 ceph-mon[101180]: pgmap v1097: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 74269/393645 objects degraded (18.867%); 790 KiB/s, 12 objects/s recovering 2024-09-14T04:20:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/466211098' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:08 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:08.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:08.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:08.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:09.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:09.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:09.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:09.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:09.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:09.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:09.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:09.499 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:09 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:20:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:09 smithi012 ceph-mon[101180]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T04:20:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 74269/393645 objects degraded (18.867%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:09.601 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:09.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[112165]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 74269/393645 objects degraded (18.867%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[107399]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T04:20:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 74269/393645 objects degraded (18.867%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:09.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:09.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:09.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:09.894 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:09.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:09.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:09.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:10.035 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:10.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:10.180 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:10.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:10.428 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.485 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:10.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:10.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:10 smithi001 ceph-mon[107399]: pgmap v1099: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 73733/389493 objects degraded (18.931%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:10 smithi001 ceph-mon[112165]: pgmap v1099: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 73733/389493 objects degraded (18.931%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:10.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:10.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:10 smithi012 ceph-mon[101180]: pgmap v1099: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 73733/389493 objects degraded (18.931%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:10.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:10.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:10.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:11.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:11.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:11.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:11.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:11.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:11.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.374 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:11.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:11.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:11.502 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:11.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:11.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:11.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:11.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:11.852 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:20:11.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:11.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:11.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:12.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:12.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:12.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:12.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:12.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:12.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:12.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:12.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:12.493 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:12 smithi001 ceph-mon[107399]: pgmap v1100: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 73216/387132 objects degraded (18.912%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:12.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:12.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:12.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:12.734 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:12.833 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:12 smithi001 ceph-mon[112165]: pgmap v1100: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 73216/387132 objects degraded (18.912%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:12.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:12 smithi012 ceph-mon[101180]: pgmap v1100: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 73216/387132 objects degraded (18.912%); 550 KiB/s, 8 objects/s recovering 2024-09-14T04:20:12.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:12.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:12.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:13.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:13.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:13.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:13.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:13.278 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:13.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:20:13.380 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.428 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:13.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:13.529 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:13.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:13.657 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:13.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:13.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:13.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:13.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:13.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:13.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:13.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:14.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:14.117 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:14.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:14.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:14.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:14.378 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:14.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:14.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:14.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:14.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:14 smithi001 ceph-mon[112165]: pgmap v1101: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 72066/380688 objects degraded (18.930%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 72066/380688 objects degraded (18.930%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:14 smithi001 ceph-mon[107399]: pgmap v1101: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 72066/380688 objects degraded (18.930%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 72066/380688 objects degraded (18.930%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:14.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:14.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:14 smithi012 ceph-mon[101180]: pgmap v1101: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 72066/380688 objects degraded (18.930%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 72066/380688 objects degraded (18.930%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:14.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:14.896 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:14.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:14.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:15.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:15.136 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:15.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:15.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:15.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:15.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:15.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:15.530 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:15.568 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:15.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:15.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:15.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:15.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:15.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:15.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:15.979 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:15.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:16.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:16.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:16.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:16.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:16.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:16.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:16.380 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:16.482 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:16.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:16.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:16.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:16.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.805 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:16.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:16 smithi012 ceph-mon[101180]: pgmap v1102: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 71533/376629 objects degraded (18.993%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:16.938 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:16.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:16.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[107399]: pgmap v1102: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 71533/376629 objects degraded (18.993%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[112165]: pgmap v1102: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 71533/376629 objects degraded (18.993%); 627 KiB/s, 9 objects/s recovering 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:16.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:16] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.51.0" 2024-09-14T04:20:17.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:17.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:17.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:17.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:17.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:17.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:17.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:17.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:17.593 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:17.615 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:17.717 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:17.740 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:17.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:17.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:17.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:17.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:18.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:18.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:18.142 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:18.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:20:18.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:20:18.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:20:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:18.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:18.342 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:18.342 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:18.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:18.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:18.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:18.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:18.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:18 smithi012 ceph-mon[101180]: pgmap v1103: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 70034/368505 objects degraded (19.005%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:20:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/691271602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:18.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:18.672 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:18 smithi001 ceph-mon[112165]: pgmap v1103: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 70034/368505 objects degraded (19.005%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:20:18.673 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/691271602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:18.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:18 smithi001 ceph-mon[107399]: pgmap v1103: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 70034/368505 objects degraded (19.005%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:20:18.673 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/691271602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:18.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:18.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:18.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:18.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:18.969 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:18.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:19.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:19.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:19.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:20:19.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:19.264 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:19.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:19.408 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:19.503 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:19.503 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:19.504 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:19.504 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 70034/368505 objects degraded (19.005%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:19.507 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:19.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:19.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:19.592 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 70034/368505 objects degraded (19.005%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:19.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 70034/368505 objects degraded (19.005%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:19.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:19.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:19.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:19.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:19.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:19.873 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:19.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:19.972 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:19.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:20.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:20.108 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:20.209 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:20.248 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:20.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.389 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:20.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:20.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:20.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:20.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:20.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:20.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:20.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:20.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:20.794 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:20.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:20.840 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:20 smithi012 ceph-mon[101180]: pgmap v1104: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 69549/364911 objects degraded (19.059%); 705 KiB/s, 11 objects/s recovering 2024-09-14T04:20:20.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:20.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:20.896 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:20.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:20.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:20.932 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[112165]: pgmap v1104: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 69549/364911 objects degraded (19.059%); 705 KiB/s, 11 objects/s recovering 2024-09-14T04:20:20.932 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:20.932 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:20.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[107399]: pgmap v1104: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 69549/364911 objects degraded (19.059%); 705 KiB/s, 11 objects/s recovering 2024-09-14T04:20:20.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:20.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:20.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-14T04:20:21.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:21.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:21.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:21.191 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:21.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:21.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:20:21.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:21.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:21.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:21.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:21.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:21.543 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:21.544 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:21.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:21.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:21.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:21.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:21.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:21.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:21.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:21.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:21.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:21.986 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:22.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:22.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:22.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:22.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:22.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:20:22.310 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:22.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:22.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:22.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:22.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:22.566 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:22.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:22.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:22 smithi001 ceph-mon[112165]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 69089/362763 objects degraded (19.045%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:20:22.630 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:22 smithi001 ceph-mon[107399]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 69089/362763 objects degraded (19.045%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:20:22.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:22.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:22.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:22.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:22 smithi012 ceph-mon[101180]: pgmap v1105: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 69089/362763 objects degraded (19.045%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:20:22.872 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:22.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:22.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:22.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:23.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:23.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:23.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:23.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:23.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:23.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:23.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:23.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:23.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:23.545 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:23.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.610 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:23.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:23.711 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:23.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:23.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:23.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:23.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:23.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:24.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:24.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:24.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:24.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:24.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:24.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:24.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:24.520 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:20:24.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:24.555 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:24.657 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:24.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:24.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:24.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:24 smithi012 ceph-mon[101180]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 68029/356697 objects degraded (19.072%); 1013 KiB/s, 15 objects/s recovering 2024-09-14T04:20:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:20:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 68029/356697 objects degraded (19.072%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:24.878 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[107399]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 68029/356697 objects degraded (19.072%); 1013 KiB/s, 15 objects/s recovering 2024-09-14T04:20:24.879 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:20:24.879 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 68029/356697 objects degraded (19.072%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:24.879 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[112165]: pgmap v1106: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 68029/356697 objects degraded (19.072%); 1013 KiB/s, 15 objects/s recovering 2024-09-14T04:20:24.879 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:20:24.879 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 68029/356697 objects degraded (19.072%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:24.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:24.953 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:24.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:25.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:25.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:25.192 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:25.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:25.325 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:25.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:25.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:25.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:25.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:25.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:20:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[107399]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T04:20:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:20:25.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:25 smithi001 ceph-mon[112165]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T04:20:25.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:25.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:25 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/752177777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:20:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:25 smithi012 ceph-mon[101180]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T04:20:25.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:25.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:25.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:26.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:26.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:26.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:26.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:26.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:26.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:26.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:26.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:26.558 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:26.558 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:26.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:26.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:26.696 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:26 smithi001 ceph-mon[107399]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 67481/352614 objects degraded (19.137%); 857 KiB/s, 13 objects/s recovering 2024-09-14T04:20:26.697 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:26 smithi001 ceph-mon[112165]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 67481/352614 objects degraded (19.137%); 857 KiB/s, 13 objects/s recovering 2024-09-14T04:20:26.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:26.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:26 smithi012 ceph-mon[101180]: pgmap v1108: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 67481/352614 objects degraded (19.137%); 857 KiB/s, 13 objects/s recovering 2024-09-14T04:20:26.855 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:26.891 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:26.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:26] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-09-14T04:20:26.992 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.033 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:27.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:27.070 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:20:27.070 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:20:27.100 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:20:27.100 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:20:27.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:27.138 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:20:27.138 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:20:27.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:27.182 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:20:27.183 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:20:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:27.213 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:20:27.213 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:20:27.246 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:20:27.246 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:20:27.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:27.285 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:20:27.285 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:20:27.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:27.327 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:20:27.327 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:20:27.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:20:27.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:27.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:27.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3193088593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:27.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:27.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3193088593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:27.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3193088593' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:27.683 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:27.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:27.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:27.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:27.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:27.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:27.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:28.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:28.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:28.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.153 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:28.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:28.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:28.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:28.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:28.387 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:28.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:20:28.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:28.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:28.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:28.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:28.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:28 smithi001 ceph-mon[107399]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 65772/343293 objects degraded (19.159%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:28 smithi001 ceph-mon[112165]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 65772/343293 objects degraded (19.159%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:28.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:28.825 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:28 smithi012 ceph-mon[101180]: pgmap v1109: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 65772/343293 objects degraded (19.159%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:28.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:28.957 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:28.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:29.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:29.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:29.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:29.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:29.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:29.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:29.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:29.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:29.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:29.597 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:29.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:180183 12.5 12:ad9d8987:::benchmark_data_smithi001_255951_object34523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.634225+0000", 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.018606826999999999, 2024-09-14T04:20:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018637322000000001, 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180183 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634225+0000", 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634225+0000", 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634224+0000", 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634231+0000", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5649999999999998e-06 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634249+0000", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8094999999999999e-05 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.634256+0000", 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7170000000000008e-06 2024-09-14T04:20:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.645954+0000", 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011697268 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.646149+0000", 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019505799999999999 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:20:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.646303+0000", 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015398500000000001 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:180121 12.5 12:a8efcf50:::benchmark_data_smithi001_255951_object34461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.568621+0000", 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.084211467999999998, 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084315267999999999, 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180121 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568621+0000", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568621+0000", 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568619+0000", 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568626+0000", 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1770000000000001e-06 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568642+0000", 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.571e-05 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.568649+0000", 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2490000000000004e-06 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.575694+0000", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070443140000000003 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for rw locks", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.575830+0000", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013660199999999999 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.629852+0000", 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054021239999999998 2024-09-14T04:20:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.629946+0000", 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4160000000000006e-05 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.630080+0000", 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000134208 2024-09-14T04:20:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.645899+0000", 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015819059 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.645914+0000", 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4521999999999999e-05 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.666 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:180187 12.5 12:a86a851e:::benchmark_data_smithi001_255951_object34527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.637354+0000", 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.015477928, 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015684266999999998, 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180187 2024-09-14T04:20:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637354+0000", 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637354+0000", 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637353+0000", 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637359+0000", 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5200000000000003e-06 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637375+0000", 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5860000000000001e-05 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.637384+0000", 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6179999999999995e-06 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.650865+0000", 2024-09-14T04:20:29.670 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 65772/343293 objects degraded (19.159%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013481065 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651070+0000", 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020560199999999999 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651235+0000", 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000164568 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:180191 12.4 12:2560f775:::benchmark_data_smithi001_255951_object34531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.644339+0000", 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0084934039999999995, 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087801270000000004, 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180191 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644339+0000", 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644339+0000", 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644337+0000", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644344+0000", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4159999999999998e-06 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644355+0000", 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.062e-05 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644362+0000", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0550000000000002e-06 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644399+0000", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6568e-05 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644541+0000", 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014226199999999999 2024-09-14T04:20:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.644655+0000", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114053 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.650986+0000", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0063312330000000003 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651007+0000", 2024-09-14T04:20:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1262000000000001e-05 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45647.0:180179 12.5 e302/298)", 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.651612+0000", 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0012206020000000001, 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00159019, 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651612+0000", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651612+0000", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651611+0000", 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:20:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651617+0000", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5289999999999997e-06 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651632+0000", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4997000000000001e-05 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.651639+0000", 2024-09-14T04:20:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8850000000000001e-06 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:180179 12.5 12:ab569583:::benchmark_data_smithi001_255951_object34519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:29.632141+0000", 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.020691509, 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021115168, 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:20:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 180179 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632141+0000", 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632141+0000", 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:20:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632140+0000", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632146+0000", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0990000000000004e-06 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632158+0000", 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2575000000000001e-05 2024-09-14T04:20:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.632166+0000", 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4660000000000002e-06 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.640391+0000", 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082251270000000005 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:20:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.640586+0000", 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019515999999999999 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:29.640802+0000", 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021642300000000001 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 6 2024-09-14T04:20:29.684 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:20:29.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:29.778 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:29.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:29.905 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:29.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 65772/343293 objects degraded (19.159%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 65772/343293 objects degraded (19.159%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:30.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:30.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:30.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:30.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:30.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:30.308 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:30.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:20:30.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:20:30.410 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:30.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:30.588 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:30.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:30 smithi012 ceph-mon[101180]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 65223/339114 objects degraded (19.233%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:30.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:30.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:30.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:30 smithi001 ceph-mon[107399]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 65223/339114 objects degraded (19.233%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:30.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:30 smithi001 ceph-mon[112165]: pgmap v1110: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 65223/339114 objects degraded (19.233%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:30.717 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-14T04:20:30.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:30.775 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:30.847 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:30.871 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:30.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:30.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:30.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:31.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:31.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:31.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:31.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:31.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:31.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:31.378 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:31.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:31.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:31.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:31.647 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:31.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:31.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:31.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:31.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:31.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:32.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:32.051 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:32.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:32.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:32.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:32.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:32.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:32.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:32.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:32.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:32.566 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:32.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:32.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.700 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:32.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:32.830 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:32.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:32 smithi012 ceph-mon[101180]: pgmap v1111: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 64703/336816 objects degraded (19.210%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:32.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:32.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:32 smithi001 ceph-mon[112165]: pgmap v1111: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 64703/336816 objects degraded (19.210%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:32.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:32 smithi001 ceph-mon[107399]: pgmap v1111: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 64703/336816 objects degraded (19.210%); 749 KiB/s, 11 objects/s recovering 2024-09-14T04:20:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:32.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:32.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:33.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:33.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:33.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:33.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:33.342 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:33.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.382 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:33.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:33.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:33.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:33.762 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:33.844 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:33.899 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:33.944 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:33.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:34.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:34.073 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:34.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:34.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:34.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:34.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:34.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:34.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:34.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:34.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:34.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:34.681 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:34 smithi001 ceph-mon[112165]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 63564/330324 objects degraded (19.243%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:20:34.682 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 63564/330324 objects degraded (19.243%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:34.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:34 smithi001 ceph-mon[107399]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 63564/330324 objects degraded (19.243%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:20:34.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 63564/330324 objects degraded (19.243%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:34.747 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:34.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:34.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:34 smithi012 ceph-mon[101180]: pgmap v1112: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 63564/330324 objects degraded (19.243%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:20:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 63564/330324 objects degraded (19.243%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:34.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:34.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:34.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:35.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:35.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:35.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:35.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:35.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:35.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:35.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:35.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.465 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:35.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:35.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:35.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:35.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:35.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:35.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:35.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:35.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:35.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:35.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:35.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:35.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:35 smithi012 ceph-mon[101180]: pgmap v1113: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 63053/326496 objects degraded (19.312%); 584 KiB/s, 9 objects/s recovering 2024-09-14T04:20:36.028 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:36.043 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:20:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:20:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:20:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:20:36.043 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[107399]: pgmap v1113: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 63053/326496 objects degraded (19.312%); 584 KiB/s, 9 objects/s recovering 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:36.080 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:35 smithi001 ceph-mon[112165]: pgmap v1113: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 63053/326496 objects degraded (19.312%); 584 KiB/s, 9 objects/s recovering 2024-09-14T04:20:36.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:36.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:36.238 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:20:36.238 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:20:36.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:36.277 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:20:36.278 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:20:36.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:36.310 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:20:36.310 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:20:36.349 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:20:36.349 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:20:36.380 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:20:36.380 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:20:36.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:36.422 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:20:36.423 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:20:36.436 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:36.456 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:20:36.456 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:20:36.490 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:20:36.490 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:20:36.538 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:36.577 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:36.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:36.716 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:36.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:36.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:20:36.865 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:36.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:36.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:36.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:36.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1768089831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1768089831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:36] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-09-14T04:20:37.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1768089831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:37.108 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:37.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:37.176 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:37.247 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:37.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:37.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:37.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:37.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:37.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:37.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:37.700 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:37.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:37.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:37.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:37.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:38.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:38.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:38.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:37 smithi001 ceph-mon[107399]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 61444/318885 objects degraded (19.268%); 848 KiB/s, 13 objects/s recovering 2024-09-14T04:20:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:37 smithi001 ceph-mon[112165]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 61444/318885 objects degraded (19.268%); 848 KiB/s, 13 objects/s recovering 2024-09-14T04:20:38.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:38.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:37 smithi012 ceph-mon[101180]: pgmap v1114: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 61444/318885 objects degraded (19.268%); 848 KiB/s, 13 objects/s recovering 2024-09-14T04:20:38.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:38.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.386 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:38.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:38.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:38.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:38.651 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:38.753 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:38.778 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:38.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:38.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:38.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:39.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:39.034 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2108078226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.034 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.036 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:20:39.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:39.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:39.153 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:39.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:39.265 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:39.265 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:39.285 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2108078226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.285 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.286 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2108078226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.286 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:20:39.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:39.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:39.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:39.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:39.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:39.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:39.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:39.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:39.750 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:39.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:39.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:39.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:39.993 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:40.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:20:40.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:40.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:40.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:40.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:40.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:40 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:40 smithi012 ceph-mon[101180]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T04:20:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:40 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 61444/318885 objects degraded (19.268%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:40 smithi012 ceph-mon[101180]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 60833/313791 objects degraded (19.386%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:40.378 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:40.411 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:40.411 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:40.411 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[112165]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T04:20:40.411 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 61444/318885 objects degraded (19.268%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:40.411 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[112165]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 60833/313791 objects degraded (19.386%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:40.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:40.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:20:40.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[107399]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T04:20:40.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 61444/318885 objects degraded (19.268%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:40.412 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:40 smithi001 ceph-mon[107399]: pgmap v1116: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 60833/313791 objects degraded (19.386%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:40.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:40.452 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:40.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:40.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:40.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:40.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:40.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:40.833 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:40.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:40.869 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:40.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:40.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-14T04:20:40.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:41.021 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:41.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:20:41.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:41.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:41.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:41.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:41.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:41.344 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:41.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:41.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:41.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:41.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:41.692 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:41.771 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:41.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:41.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:41.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:42.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:42.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:20:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:42.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:42.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:42.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:42.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:42.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:42.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:42.459 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:42.459 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:42.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:42.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:42.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:42.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:42.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:42 smithi001 ceph-mon[112165]: pgmap v1117: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 60324/311517 objects degraded (19.365%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:42 smithi001 ceph-mon[107399]: pgmap v1117: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 60324/311517 objects degraded (19.365%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:42.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:42.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:42.844 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:42 smithi012 ceph-mon[101180]: pgmap v1117: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 60324/311517 objects degraded (19.365%); 716 KiB/s, 11 objects/s recovering 2024-09-14T04:20:42.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:42.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:42.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:43.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:43.110 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:43.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:43.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:43.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:43.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:20:43.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:43.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:43.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:43.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:43.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:43.545 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:43.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:43.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:43.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:43.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:43.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:43.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:43.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:43.962 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:44.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:44.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:44.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:44.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:44.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:44.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:44.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:44.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:44.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:44.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:44.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:44.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:44 smithi001 ceph-mon[107399]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 59226/305166 objects degraded (19.408%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 59226/305166 objects degraded (19.408%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:44 smithi001 ceph-mon[112165]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 59226/305166 objects degraded (19.408%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 59226/305166 objects degraded (19.408%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:44.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:44.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:44.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:44 smithi012 ceph-mon[101180]: pgmap v1118: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 59226/305166 objects degraded (19.408%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 59226/305166 objects degraded (19.408%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:44.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:44.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:44.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:45.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:45.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:45.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:45.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:45.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:45.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:45.372 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:45.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:45.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:45.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:45.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:45.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:45.778 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:45.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:45.833 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:45.833 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:45.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:45.968 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:45.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:46.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:46.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:46.205 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:46.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:46.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:46.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:46.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:46.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:46 smithi001 ceph-mon[107399]: pgmap v1119: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 58689/301158 objects degraded (19.488%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:46.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:46 smithi001 ceph-mon[112165]: pgmap v1119: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 58689/301158 objects degraded (19.488%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:46.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.515 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:46.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:46.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:46 smithi012 ceph-mon[101180]: pgmap v1119: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 58689/301158 objects degraded (19.488%); 691 KiB/s, 10 objects/s recovering 2024-09-14T04:20:46.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:46.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:46.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:20:46.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:46.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:46.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:46.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:46.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:46.961 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:46] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-09-14T04:20:47.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:47.061 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:20:47.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:47.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.210 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:47.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:47.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:47.351 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:47.453 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:47.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:47.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:47.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:47.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:47.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:47.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:47.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:47.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:48.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:48.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:48.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:48.183 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:48.285 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:48.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:48.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:48.451 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:48.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:48.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:48.647 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:48 smithi012 ceph-mon[101180]: pgmap v1120: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 57154/293433 objects degraded (19.478%); 685 KiB/s, 10 objects/s recovering 2024-09-14T04:20:48.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:48.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:48 smithi001 ceph-mon[107399]: pgmap v1120: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 57154/293433 objects degraded (19.478%); 685 KiB/s, 10 objects/s recovering 2024-09-14T04:20:48.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:48 smithi001 ceph-mon[112165]: pgmap v1120: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 57154/293433 objects degraded (19.478%); 685 KiB/s, 10 objects/s recovering 2024-09-14T04:20:48.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:48.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-14T04:20:48.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:48.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:48.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:48.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:49.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:49.137 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:20:49.137 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:20:49.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:49.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:49.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:49.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:49.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:49.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:49.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:49.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:49.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 57154/293433 objects degraded (19.478%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 57154/293433 objects degraded (19.478%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:49.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 57154/293433 objects degraded (19.478%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T04:20:49.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:49.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:49.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:49.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:50.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:20:50.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:50.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:50.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:50.276 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:20:50.276 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:20:50.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:50.330 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:20:50.330 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:20:50.374 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:20:50.374 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:20:50.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:50.410 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:20:50.410 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:20:50.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:50.447 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:20:50.447 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:20:50.461 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:50.462 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:50.499 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:20:50.499 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:20:50.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:50.540 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:20:50.540 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:20:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:50.582 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[107399]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 56585/288414 objects degraded (19.619%); 662 KiB/s, 10 objects/s recovering 2024-09-14T04:20:50.582 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:50.582 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2279535411' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:50.582 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[112165]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 56585/288414 objects degraded (19.619%); 662 KiB/s, 10 objects/s recovering 2024-09-14T04:20:50.583 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:50.583 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2279535411' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:50.585 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:20:50.585 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:20:50.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:50.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:50.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:50 smithi012 ceph-mon[101180]: pgmap v1121: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 56585/288414 objects degraded (19.619%); 662 KiB/s, 10 objects/s recovering 2024-09-14T04:20:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2279535411' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:20:50.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:50.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:50.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:51.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:51.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:51.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:51.229 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:51.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:51.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:51.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:51.477 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:20:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:51.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:51.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:51.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:51.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:51.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:51.776 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:51.819 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3912418555' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.820 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:51 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3912418555' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.820 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:51 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3912418555' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:51 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:20:51.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:51.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:52.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:52.052 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:52.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:52.208 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:52.310 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:52.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:52.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:52.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:52.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:52.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:52.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:52.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:52 smithi012 ceph-mon[101180]: pgmap v1122: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 55765/284124 objects degraded (19.627%); 720 KiB/s, 11 objects/s recovering 2024-09-14T04:20:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:52 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:20:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:52 smithi012 ceph-mon[101180]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T04:20:52.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:52.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:52.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[107399]: pgmap v1122: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 55765/284124 objects degraded (19.627%); 720 KiB/s, 11 objects/s recovering 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[107399]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[112165]: pgmap v1122: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 55765/284124 objects degraded (19.627%); 720 KiB/s, 11 objects/s recovering 2024-09-14T04:20:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:20:52.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:20:52.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:52 smithi001 ceph-mon[112165]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T04:20:52.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:53.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:53.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.169 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:53.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:53.270 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:53.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:53.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:53.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:53.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:53.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:53.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:53.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:53.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:53.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:53.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:53.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:20:53.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:53.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:53.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:54.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:54.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:54.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:54.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:54.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:54.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:54.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:54.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:54.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:54.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:54.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:54.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:54 smithi012 ceph-mon[101180]: pgmap v1124: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 55212/281754 objects degraded (19.596%); 557 KiB/s, 8 objects/s recovering 2024-09-14T04:20:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 55212/281754 objects degraded (19.596%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:54.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:54.909 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:54 smithi001 ceph-mon[112165]: pgmap v1124: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 55212/281754 objects degraded (19.596%); 557 KiB/s, 8 objects/s recovering 2024-09-14T04:20:54.909 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 55212/281754 objects degraded (19.596%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:54.910 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:54 smithi001 ceph-mon[107399]: pgmap v1124: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 55212/281754 objects degraded (19.596%); 557 KiB/s, 8 objects/s recovering 2024-09-14T04:20:54.910 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 55212/281754 objects degraded (19.596%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:54.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:54.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:54.987 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:55.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:55.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:55.128 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:20:55.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:55.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:55.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.331 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:55.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:55.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.474 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:55.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:55.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.606 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:55.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:55.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:55.746 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:55.848 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:55.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:55.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:55.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:56.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:56.067 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:20:56.067 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:202534 12.a 12:529d55ed:::benchmark_data_smithi001_255951_object56874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:56.060530+0000", 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0051078190000000004, 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051389260000000003, 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:20:56.068 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 202534 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060530+0000", 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060530+0000", 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:20:56.069 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060531+0000", 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5409999999999999e-06 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060536+0000", 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6430000000000003e-06 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:20:56.070 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060547+0000", 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0529e-05 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060553+0000", 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8159999999999996e-06 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060626+0000", 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2816000000000002e-05 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.071 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060745+0000", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118397 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.060877+0000", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000132473 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:202540 12.6 12:6a3ef47b:::benchmark_data_smithi001_255951_object56880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:56.063296+0000", 2024-09-14T04:20:56.072 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0023413549999999998, 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024430419999999999, 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 202540 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063296+0000", 2024-09-14T04:20:56.073 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063296+0000", 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063298+0000", 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.249e-06 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:20:56.074 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063302+0000", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3780000000000003e-06 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063313+0000", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0971999999999999e-05 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063320+0000", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4680000000000003e-06 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:20:56.075 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063408+0000", 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7278e-05 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063611+0000", 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000203224 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:56.063720+0000", 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000109468 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:20:56.076 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:20:56.077 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:20:56.077 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:20:56.077 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 2 2024-09-14T04:20:56.077 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:20:56.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:56.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:56.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:56.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:56.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:56.433 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:56.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:56.564 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:56.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:56.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:56.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:56.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:56 smithi012 ceph-mon[101180]: pgmap v1125: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 53791/271860 objects degraded (19.786%); 883 KiB/s, 13 objects/s recovering 2024-09-14T04:20:56.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:56.954 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:56 smithi001 ceph-mon[107399]: pgmap v1125: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 53791/271860 objects degraded (19.786%); 883 KiB/s, 13 objects/s recovering 2024-09-14T04:20:56.954 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:56 smithi001 ceph-mon[112165]: pgmap v1125: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 53791/271860 objects degraded (19.786%); 883 KiB/s, 13 objects/s recovering 2024-09-14T04:20:56.956 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:20:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:20:56] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-09-14T04:20:56.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:56.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:56.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:20:57.071 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:57.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:57.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:57.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:57.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:57.384 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:57.486 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:57.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:57.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.660 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:57.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:57.742 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-14T04:20:57.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:57.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:57.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:57.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:57.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:58.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:58.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:58.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:20:58.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:20:58.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:58.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:58.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:58.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:58.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:20:58.648 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:20:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:58 smithi001 ceph-mon[107399]: pgmap v1126: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 52991/268599 objects degraded (19.729%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:20:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:58 smithi001 ceph-mon[112165]: pgmap v1126: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 52991/268599 objects degraded (19.729%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:20:58.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:58.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:58.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:58.803 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:58.808 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:58 smithi012 ceph-mon[101180]: pgmap v1126: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 52991/268599 objects degraded (19.729%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:20:58.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:58.896 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:20:58.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:58.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:59.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:20:59.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:59.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:20:59.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:20:59.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:20:59.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:20:59.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:20:59.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:20:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:59.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:20:59.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:59.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:59.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:20:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:20:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 52991/268599 objects degraded (19.729%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:20:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 52991/268599 objects degraded (19.729%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:59.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:20:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:20:59.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:20:59.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:20:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 52991/268599 objects degraded (19.729%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:20:59.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:20:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:20:59.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:20:59.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:00.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:00.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:00.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.302 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:00.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:00.435 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:00.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:00.567 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:00.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:00.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:00.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:00.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:00 smithi012 ceph-mon[101180]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 52469/264711 objects degraded (19.821%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:21:00.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:00.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:00 smithi001 ceph-mon[107399]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 52469/264711 objects degraded (19.821%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:21:00.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:00 smithi001 ceph-mon[112165]: pgmap v1127: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 52469/264711 objects degraded (19.821%); 505 KiB/s, 7 objects/s recovering 2024-09-14T04:21:00.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:00.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:01.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:01.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:01.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:01.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:01.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:01.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:01.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:01.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:01.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.666 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:01.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:01.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:01.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:01.904 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:01.942 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:01.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:02.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.072 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:02.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:02.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:02.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:02.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.354 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:02.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:02.455 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:02.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:02.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:02.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:02.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:02.759 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:02 smithi012 ceph-mon[101180]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 51324/258177 objects degraded (19.879%); 672 KiB/s, 10 objects/s recovering 2024-09-14T04:21:02.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:02.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:02.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:02.900 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:02 smithi001 ceph-mon[112165]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 51324/258177 objects degraded (19.879%); 672 KiB/s, 10 objects/s recovering 2024-09-14T04:21:02.900 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:02 smithi001 ceph-mon[107399]: pgmap v1128: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 51324/258177 objects degraded (19.879%); 672 KiB/s, 10 objects/s recovering 2024-09-14T04:21:02.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:03.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:03.125 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:03.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:03.261 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:03.302 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:03.404 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:03.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:03.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:03.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:03.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:03.705 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:21:03.705 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:21:03.707 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:03.745 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:21:03.746 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:21:03.782 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:21:03.782 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:21:03.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:03.818 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:21:03.818 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:21:03.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:03.851 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:21:03.851 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:21:03.879 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:21:03.879 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:21:03.910 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:21:03.910 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:21:03.941 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:21:03.941 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:21:03.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:03.977 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:03.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:04.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:04.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:04.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:04.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:04.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:04.374 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:04.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:04.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:04.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:04.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:04.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:04.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:04.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:04 smithi012 ceph-mon[101180]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 50768/255702 objects degraded (19.854%); 564 KiB/s, 8 objects/s recovering 2024-09-14T04:21:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:04 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2582092032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 50768/255702 objects degraded (19.854%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:04.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:04.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:04.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:04.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[107399]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 50768/255702 objects degraded (19.854%); 564 KiB/s, 8 objects/s recovering 2024-09-14T04:21:04.929 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2582092032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:04.929 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 50768/255702 objects degraded (19.854%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:04.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[112165]: pgmap v1129: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 50768/255702 objects degraded (19.854%); 564 KiB/s, 8 objects/s recovering 2024-09-14T04:21:04.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2582092032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:04.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 50768/255702 objects degraded (19.854%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:05.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:05.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:05.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:05.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:05.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:05.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:05.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:05.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:05.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:05.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:05.613 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:05.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:05.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:05.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:05.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:05.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:05.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:05.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:05.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:06.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:06.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:06.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:06.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:06.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:06.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:06.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:06.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:06.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:06.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:06.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:06.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:06.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:06 smithi012 ceph-mon[101180]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 49324/245490 objects degraded (20.092%); 794 KiB/s, 12 objects/s recovering 2024-09-14T04:21:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:06.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:06.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:06 smithi001 ceph-mon[112165]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 49324/245490 objects degraded (20.092%); 794 KiB/s, 12 objects/s recovering 2024-09-14T04:21:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:06 smithi001 ceph-mon[107399]: pgmap v1130: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 49324/245490 objects degraded (20.092%); 794 KiB/s, 12 objects/s recovering 2024-09-14T04:21:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:06] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-09-14T04:21:06.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:06.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:07.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:07.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:07.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:07.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:07.363 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:07.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:07.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:07.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:07.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:07.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:07.666 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:07.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:07.804 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:07.906 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:07.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:07.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:08.044 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:08.073 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:08.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.213 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:08.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:08.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:08.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:08.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:08.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:08.576 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:08 smithi001 ceph-mon[107399]: pgmap v1131: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 48544/242310 objects degraded (20.034%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:08.576 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:08 smithi001 ceph-mon[112165]: pgmap v1131: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 48544/242310 objects degraded (20.034%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:08.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:08.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:08.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:08.759 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:08 smithi012 ceph-mon[101180]: pgmap v1131: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 48544/242310 objects degraded (20.034%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:08.861 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:08.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:08.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:09.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:09.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:21:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:09.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:09.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:09.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:09.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:09.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:09.441 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:09.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:09.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:09.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:09.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.771 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:09.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 48544/242310 objects degraded (20.034%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:09.872 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:09.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:09.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 48544/242310 objects degraded (20.034%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:09.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 48544/242310 objects degraded (20.034%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:10.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:10.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:10.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:10.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:10.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:10.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:10.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.503 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:10.572 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:21:10.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:10.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:10.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:10.782 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:10 smithi012 ceph-mon[101180]: pgmap v1132: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 47968/238227 objects degraded (20.135%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3006297764' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:10 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:10.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:10.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:10.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[112165]: pgmap v1132: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 47968/238227 objects degraded (20.135%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3006297764' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[107399]: pgmap v1132: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 47968/238227 objects degraded (20.135%); 522 KiB/s, 8 objects/s recovering 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3006297764' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:10 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:11.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:11.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:11.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:11.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:11.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-14T04:21:11.315 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:11.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:11.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:11.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:11.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:11.527 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:21:11.527 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:21:11.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:11.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:11.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:11.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:11.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:11 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:21:11.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:11 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:11.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:11 smithi012 ceph-mon[101180]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T04:21:11.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:11.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:11.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[112165]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:11.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:11 smithi001 ceph-mon[107399]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T04:21:12.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:12.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:12.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:12.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:12.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:12.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:12.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:12.339 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:12.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:12.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:12.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:12.887 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:12 smithi012 ceph-mon[101180]: pgmap v1134: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 46792/231636 objects degraded (20.201%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:12.888 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:12.889 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:12 smithi001 ceph-mon[107399]: pgmap v1134: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 46792/231636 objects degraded (20.201%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:12.889 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:12 smithi001 ceph-mon[112165]: pgmap v1134: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 46792/231636 objects degraded (20.201%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:12.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:12.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:12.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:21:13.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:13.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:13.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:13.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:13.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:13.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:13.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:13.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:13.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:13.661 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:13.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:13.771 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:13.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:13.821 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:13.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:13.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:13.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:14.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:14.054 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:14.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:14.184 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:14.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:14.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:14.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:14.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:14.547 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:21:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=infra.usagestats t=2024-09-14T04:21:14.282434918Z level=info msg="Usage stats are ready to report" 2024-09-14T04:21:14.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:14.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:14.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:14 smithi001 ceph-mon[112165]: pgmap v1135: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 46275/229311 objects degraded (20.180%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 46275/229311 objects degraded (20.180%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:14 smithi001 ceph-mon[107399]: pgmap v1135: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 46275/229311 objects degraded (20.180%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 46275/229311 objects degraded (20.180%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:14.733 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:14.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:14 smithi012 ceph-mon[101180]: pgmap v1135: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 46275/229311 objects degraded (20.180%); 588 KiB/s, 9 objects/s recovering 2024-09-14T04:21:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 46275/229311 objects degraded (20.180%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:14.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:14.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:14.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:14.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:21:14.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.001 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:15.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:15.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:15.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:15.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.280 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:15.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:15.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:15.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:15.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:15.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:15.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:15.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:15.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:15.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:15.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:15.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:15.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:16.063 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:16.093 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:16.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:16.223 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:16.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:16.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.353 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:16.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:16.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:16.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:16.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:16.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:16.718 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.762 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:16.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:16 smithi012 ceph-mon[101180]: pgmap v1136: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 44956/220257 objects degraded (20.411%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:16.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:16.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:16.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:16 smithi001 ceph-mon[107399]: pgmap v1136: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 44956/220257 objects degraded (20.411%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:16 smithi001 ceph-mon[112165]: pgmap v1136: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 44956/220257 objects degraded (20.411%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:16] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-09-14T04:21:16.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:17.020 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:17.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:17.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:17.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:17.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:17.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:17.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:17.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:17.433 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:17.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:17.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:17.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:17.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:17.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.848 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:17.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:17.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:17.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:17.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:18.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:18.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:18.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:18.261 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:18.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:18.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:18.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:18.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:18.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:18.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:18.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:18.778 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:18 smithi012 ceph-mon[101180]: pgmap v1137: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 44200/217299 objects degraded (20.341%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:18.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:18.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:18.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:18.941 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:18 smithi001 ceph-mon[107399]: pgmap v1137: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 44200/217299 objects degraded (20.341%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:18.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:18 smithi001 ceph-mon[112165]: pgmap v1137: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 44200/217299 objects degraded (20.341%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:19.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:19.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:19.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:19.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:19.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:19.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:19.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:19.479 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:19.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:19.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:19.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:19.752 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:19.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:19.764 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 44200/217299 objects degraded (20.341%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:19.764 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:19.764 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 44200/217299 objects degraded (20.341%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 44200/217299 objects degraded (20.341%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:19.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:19.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:19.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:19.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:20.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:20.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:20.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:20.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:20.304 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:21:20.304 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:21:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:20.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:20.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:20.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:20.632 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[107399]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43704/213486 objects degraded (20.472%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:20.632 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:20.632 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:20.632 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1965638390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:20.632 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[112165]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43704/213486 objects degraded (20.472%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:20.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:20.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:20.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1965638390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:20.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:20.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:20.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:20 smithi012 ceph-mon[101180]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43704/213486 objects degraded (20.472%); 646 KiB/s, 10 objects/s recovering 2024-09-14T04:21:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1965638390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:20.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:20.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:20.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:20.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:21.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:21.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:21.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:21.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:21.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:21.357 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:21:21.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:21:21.359 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:21:21.359 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:21:21.359 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:21:21.359 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:21:21.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:21.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:21.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:21.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:21.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:21:21.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:21.609 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:21 smithi001 ceph-mon[112165]: pgmap v1139: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 42562/206943 objects degraded (20.567%); 715 KiB/s, 11 objects/s recovering 2024-09-14T04:21:21.609 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:21 smithi001 ceph-mon[107399]: pgmap v1139: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 42562/206943 objects degraded (20.567%); 715 KiB/s, 11 objects/s recovering 2024-09-14T04:21:21.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:21.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:21.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:21.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:21.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:21 smithi012 ceph-mon[101180]: pgmap v1139: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 42562/206943 objects degraded (20.567%); 715 KiB/s, 11 objects/s recovering 2024-09-14T04:21:21.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:21.980 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:21.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:22.082 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:22.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:22.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:22.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:22.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:22.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:22.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:22.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:22.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.712 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:22.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:22.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:22.850 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:22.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:22.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:22.970 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:23.017 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:23.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:23.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:23.157 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:23.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:23.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:23.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:23.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:23.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:23.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:23.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:23.707 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:23.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:23.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:23.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:23.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:23.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:23.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:24.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:24.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:24.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:21:24.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:24.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:24.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:24.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:24.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:24.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:24.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:24.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[112165]: pgmap v1140: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 42030/204648 objects degraded (20.538%); 645 KiB/s, 10 objects/s recovering 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 42030/204648 objects degraded (20.538%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[107399]: pgmap v1140: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 42030/204648 objects degraded (20.538%); 645 KiB/s, 10 objects/s recovering 2024-09-14T04:21:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 42030/204648 objects degraded (20.538%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:24.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:24 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:24.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:24 smithi012 ceph-mon[101180]: pgmap v1140: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 42030/204648 objects degraded (20.538%); 645 KiB/s, 10 objects/s recovering 2024-09-14T04:21:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 42030/204648 objects degraded (20.538%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:24 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:24 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:24.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:24.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:24.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:24.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:25.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:25.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:25.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:25.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:25.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:25.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:25.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:25.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:25.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:25.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:25.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:25.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:25.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:25.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:25.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:26.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:26.108 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:26.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:26.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:26.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:26.368 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:26.470 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:26.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:26.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:26.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:26.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:26 smithi001 ceph-mon[112165]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 40670/195357 objects degraded (20.818%); 922 KiB/s, 14 objects/s recovering 2024-09-14T04:21:26.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:26 smithi001 ceph-mon[107399]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 40670/195357 objects degraded (20.818%); 922 KiB/s, 14 objects/s recovering 2024-09-14T04:21:26.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:26] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-09-14T04:21:26.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:26.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:26 smithi012 ceph-mon[101180]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 40670/195357 objects degraded (20.818%); 922 KiB/s, 14 objects/s recovering 2024-09-14T04:21:26.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:26.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:26.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:27.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:27.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:27.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:27.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:27.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:27.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:27.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:27.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:27.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:27.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:27.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:27.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:27.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:27.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:27.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:27.869 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:27.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:27.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:27.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:27.932 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2024-09-14T04:21:27.932 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:21:27.932 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-14T04:21:27.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd out 6 2024-09-14T04:21:27.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:28.072 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:28.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:28.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:28.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:28.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:28.468 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:21:28.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:28.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:28.633 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:28.672 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[112165]: pgmap v1142: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 39831/192000 objects degraded (20.745%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1314851387' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[107399]: pgmap v1142: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 39831/192000 objects degraded (20.745%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1314851387' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:28 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230410 12.7 e306/298 12:e8eed388:::benchmark_data_smithi001_255951_object84750:head v 306'14573, mlcod=306'14570)", 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.774578+0000", 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0088726440000000007, 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089013740000000001, 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774578+0000", 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774578+0000", 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774579+0000", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4000000000000001e-07 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774584+0000", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6319999999999999e-06 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774592+0000", 2024-09-14T04:21:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1470000000000001e-06 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774596+0000", 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4290000000000003e-06 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774758+0000", 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016155400000000001 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.774777+0000", 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9199e-05 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230415 12.7 e306/298 12:ee388e50:::benchmark_data_smithi001_255951_object84755:head v 306'14574, mlcod=306'14570)", 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.777766+0000", 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0056849969999999998, 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057686200000000003, 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777766+0000", 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777766+0000", 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777767+0000", 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6600000000000005e-07 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777773+0000", 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5740000000000003e-06 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777783+0000", 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0166999999999999e-05 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777789+0000", 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6409999999999996e-06 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.779682+0000", 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001892857 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.779701+0000", 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8423e-05 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230416 12.d e306/298)", 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.778089+0000", 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0053619990000000001, 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0054905780000000003, 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778089+0000", 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778089+0000", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778090+0000", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3000000000000001e-07 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778094+0000", 2024-09-14T04:21:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2350000000000001e-06 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778102+0000", 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0630000000000006e-06 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.778108+0000", 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.66e-06 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230403 12.d e306/298 12:b0efb4f1:::benchmark_data_smithi001_255951_object84743:head v 306'14601, mlcod=306'14598)", 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.767238+0000", 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.016213538, 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016383037, 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767238+0000", 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767238+0000", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767238+0000", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5300000000000004e-07 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767244+0000", 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.322e-06 2024-09-14T04:21:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767260+0000", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6501e-05 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.767267+0000", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2259999999999999e-06 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.768177+0000", 2024-09-14T04:21:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00091001299999999999 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.768195+0000", 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8114999999999999e-05 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230409 12.7 e306/298 12:e10a1a1b:::benchmark_data_smithi001_255951_object84749:head v 306'14572, mlcod=306'14570)", 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.770920+0000", 2024-09-14T04:21:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.012531456999999999, 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012749065, 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770920+0000", 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770920+0000", 2024-09-14T04:21:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770921+0000", 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5600000000000004e-07 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770928+0000", 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6359999999999994e-06 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770940+0000", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1552999999999999e-05 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.770955+0000", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5235999999999999e-05 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771025+0000", 2024-09-14T04:21:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9930000000000003e-05 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771043+0000", 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7923000000000001e-05 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230407 12.5 e306/298 12:a26fc5d5:::benchmark_data_smithi001_255951_object84747:head v 306'14402, mlcod=289'5272)", 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.771133+0000", 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.01231811, 2024-09-14T04:21:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012582059, 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771133+0000", 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771133+0000", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771133+0000", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8299999999999998e-07 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771137+0000", 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7720000000000001e-06 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771144+0000", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4899999999999997e-06 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.771149+0000", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1950000000000002e-06 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.775415+0000", 2024-09-14T04:21:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042660950000000001 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.775477+0000", 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1808000000000002e-05 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:230404 12.a e306/298 12:5e537ebb:::benchmark_data_smithi001_255951_object84744:head v 306'14638, mlcod=306'14635)", 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:21:28.773619+0000", 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.009832337, 2024-09-14T04:21:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010143252, 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773619+0000", 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773619+0000", 2024-09-14T04:21:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773620+0000", 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1540000000000001e-06 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773625+0000", 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4530000000000001e-06 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773636+0000", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0265000000000001e-05 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.773643+0000", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9040000000000003e-06 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777421+0000", 2024-09-14T04:21:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003778048 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:21:28.777449+0000", 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7900000000000001e-05 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 7 2024-09-14T04:21:28.814 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:21:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:28 smithi012 ceph-mon[101180]: pgmap v1142: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 39831/192000 objects degraded (20.745%); 640 KiB/s, 9 objects/s recovering 2024-09-14T04:21:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1314851387' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:28 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:21:28.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:28.952 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:28.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:29.055 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:29.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:29.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:29.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:29.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:29.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:29.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:29.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:29.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:29.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:29.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:21:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[107399]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T04:21:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 39831/192000 objects degraded (20.745%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:21:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[112165]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T04:21:29.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 39831/192000 objects degraded (20.745%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:29.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:29.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:29 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:29 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:21:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:29 smithi012 ceph-mon[101180]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T04:21:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 39831/192000 objects degraded (20.745%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:29.869 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:29.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:29.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:30.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:30.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:30.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:30.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:30.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:30.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:21:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:30.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:30.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:30.602 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:30.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:30.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:30 smithi001 ceph-mon[107399]: pgmap v1144: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 39322/188073 objects degraded (20.908%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:21:30.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:30 smithi001 ceph-mon[112165]: pgmap v1144: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 39322/188073 objects degraded (20.908%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:21:30.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:30.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:30 smithi012 ceph-mon[101180]: pgmap v1144: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 39322/188073 objects degraded (20.908%); 768 KiB/s, 11 objects/s recovering 2024-09-14T04:21:30.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:30.960 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:30.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:30.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:31.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:31.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:31.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:31.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:31.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:31.364 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:31.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:31.502 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:31.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:31.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:31.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:31.768 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:31.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:31.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:31.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:31.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:31.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:32.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:32.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:32.132 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:32.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:32.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:32.325 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:32.347 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.361 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:32.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:32.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:32.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:32.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:32.628 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:32.730 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:32.739 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:32 smithi001 ceph-mon[107399]: pgmap v1145: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 38199/181575 objects degraded (21.038%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:32.739 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:32 smithi001 ceph-mon[112165]: pgmap v1145: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 38199/181575 objects degraded (21.038%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:32.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:32.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:32 smithi012 ceph-mon[101180]: pgmap v1145: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 38199/181575 objects degraded (21.038%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:32.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:32.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:32.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:33.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:33.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:33.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:33.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:33.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:33.312 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:21:33.312 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:21:33.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:33.349 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:21:33.349 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:21:33.384 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:21:33.385 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:21:33.421 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:21:33.422 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:21:33.425 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:33.455 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:21:33.455 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:21:33.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:33.488 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:21:33.488 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:21:33.519 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:21:33.519 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:21:33.556 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:21:33.556 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:21:33.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:33.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:33.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:33.724 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:33 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1764703841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:33.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:33 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1764703841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:33.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:33.782 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:21:33.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:33 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1764703841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:33.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:33.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:33.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:33.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:34.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:34.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:34.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:34.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:34.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:34.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:34.433 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:34.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:34.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:34.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:34.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:34.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:34.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:34 smithi012 ceph-mon[101180]: pgmap v1146: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 37660/179196 objects degraded (21.016%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 37660/179196 objects degraded (21.016%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:34.850 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:34.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:34 smithi001 ceph-mon[112165]: pgmap v1146: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 37660/179196 objects degraded (21.016%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 37660/179196 objects degraded (21.016%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:34 smithi001 ceph-mon[107399]: pgmap v1146: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 37660/179196 objects degraded (21.016%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:21:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 37660/179196 objects degraded (21.016%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:34.976 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:34.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:35.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:35.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:35.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:35.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:35.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:35.386 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:35.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:35.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:35.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:35.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:35.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:35.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:35.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:35.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:35.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:35.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:35.928 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:35.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:35.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:36.030 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:36.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:36.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:36.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[112165]: pgmap v1147: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 36292/169926 objects degraded (21.358%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:35 smithi001 ceph-mon[107399]: pgmap v1147: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 36292/169926 objects degraded (21.358%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:36.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:36.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:35 smithi012 ceph-mon[101180]: pgmap v1147: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 36292/169926 objects degraded (21.358%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:36.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:36.475 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:36.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:36.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:36.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.741 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:36.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:36.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:36.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:36.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:36] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-09-14T04:21:36.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:37.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:37.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:37.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:37.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.279 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:37.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:37.380 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:37.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:37.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:37.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:37.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:37.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:37.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:37.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:37.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:37.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:37.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:38.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:38.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:38.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:38.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:38.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:38.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:38.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:38.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:38.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:38.630 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:38.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:38.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:38 smithi001 ceph-mon[107399]: pgmap v1148: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 35487/166689 objects degraded (21.289%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:38.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:38 smithi001 ceph-mon[112165]: pgmap v1148: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 35487/166689 objects degraded (21.289%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:38.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:38.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:38 smithi012 ceph-mon[101180]: pgmap v1148: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 35487/166689 objects degraded (21.289%); 742 KiB/s, 11 objects/s recovering 2024-09-14T04:21:38.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:38.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:38.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:39.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:39.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:39.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:39.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:39.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:39.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:39.432 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:21:39.432 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:21:39.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:39.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:39.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:39.620 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:39.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 35487/166689 objects degraded (21.289%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:39.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 35487/166689 objects degraded (21.289%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:39.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:39.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 35487/166689 objects degraded (21.289%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:39.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:39.890 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:39.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:39.991 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:40.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:40.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:40.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:21:40.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:40.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:40.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:40.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:40.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:40.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:40.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:40.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:40.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:40.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:40.828 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:40 smithi001 ceph-mon[107399]: pgmap v1149: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 34907/162462 objects degraded (21.486%); 679 KiB/s, 10 objects/s recovering 2024-09-14T04:21:40.829 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:40 smithi001 ceph-mon[112165]: pgmap v1149: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 34907/162462 objects degraded (21.486%); 679 KiB/s, 10 objects/s recovering 2024-09-14T04:21:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:40 smithi012 ceph-mon[101180]: pgmap v1149: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 34907/162462 objects degraded (21.486%); 679 KiB/s, 10 objects/s recovering 2024-09-14T04:21:40.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:40.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:40.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:40.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:41.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:41.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:41.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:41.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:41.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:41.319 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:41.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:41.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:41.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:41.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:41.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:21:41.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:41.712 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:41.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:41.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:41.951 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:41.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:42.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:42.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:42.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:42.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:42.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:42.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:42.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:21:42.274+0000 7fbfffcba640 -1 mon.a@0(leader).osd e306 definitely_dead 0 2024-09-14T04:21:42.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:42.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:42.494 INFO:teuthology.orchestra.run.smithi001.stderr:marked out osd.6. 2024-09-14T04:21:42.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:42.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:42.709 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[107399]: pgmap v1150: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 33848/156069 objects degraded (21.688%); 863 KiB/s, 13 objects/s recovering 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/745520341' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[107399]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[112165]: pgmap v1150: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 33848/156069 objects degraded (21.688%); 863 KiB/s, 13 objects/s recovering 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/745520341' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:42 smithi001 ceph-mon[112165]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-14T04:21:42.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:42.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:42 smithi012 ceph-mon[101180]: pgmap v1150: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 33848/156069 objects degraded (21.688%); 863 KiB/s, 13 objects/s recovering 2024-09-14T04:21:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:42 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/745520341' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-14T04:21:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:42 smithi012 ceph-mon[101180]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-14T04:21:42.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:42.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:42.983 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:42.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:43.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:43.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.109 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:43.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:43.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:43.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:43.336 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:43.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:43.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:43.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:43.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:43.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:43.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:43.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[112165]: osdmap e307: 8 total, 8 up, 7 in 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[112165]: osdmap e308: 8 total, 8 up, 7 in 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[107399]: osdmap e307: 8 total, 8 up, 7 in 2024-09-14T04:21:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:43 smithi001 ceph-mon[107399]: osdmap e308: 8 total, 8 up, 7 in 2024-09-14T04:21:43.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:43.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:43 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-14T04:21:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:43 smithi012 ceph-mon[101180]: osdmap e307: 8 total, 8 up, 7 in 2024-09-14T04:21:43.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:43 smithi012 ceph-mon[101180]: osdmap e308: 8 total, 8 up, 7 in 2024-09-14T04:21:43.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:43.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:43.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:44.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:44.034 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:44.136 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:44.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:44.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:44.287 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:44.389 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:44.415 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:44.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:44.551 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:44.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:44.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[107399]: pgmap v1152: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 33323/153690 objects degraded (21.682%); 659 KiB/s, 10 objects/s recovering 2024-09-14T04:21:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 33323/153690 objects degraded (21.682%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[107399]: osdmap e309: 8 total, 8 up, 7 in 2024-09-14T04:21:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[112165]: pgmap v1152: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 33323/153690 objects degraded (21.682%); 659 KiB/s, 10 objects/s recovering 2024-09-14T04:21:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 33323/153690 objects degraded (21.682%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:44 smithi001 ceph-mon[112165]: osdmap e309: 8 total, 8 up, 7 in 2024-09-14T04:21:44.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:44.811 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:44 smithi012 ceph-mon[101180]: pgmap v1152: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 100 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 33323/153690 objects degraded (21.682%); 659 KiB/s, 10 objects/s recovering 2024-09-14T04:21:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 33323/153690 objects degraded (21.682%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:44 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:44 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:44.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:44 smithi012 ceph-mon[101180]: osdmap e309: 8 total, 8 up, 7 in 2024-09-14T04:21:44.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:44.936 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:44.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:45.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:45.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:45.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:45.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.186 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:45.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:45.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:45.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:45.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:45.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:45.445 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:45.491 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:21:45.506 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:45.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:45.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:45.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:45.700 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15355 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 306'15419 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15157 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[107399]: osdmap e310: 8 total, 8 up, 7 in 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15355 2024-09-14T04:21:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 306'15419 2024-09-14T04:21:45.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15157 2024-09-14T04:21:45.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:45.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:45.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:45.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:45 smithi001 ceph-mon[112165]: osdmap e310: 8 total, 8 up, 7 in 2024-09-14T04:21:45.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:45.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15355 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 306'15419 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 306'15157 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2755412449' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:21:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:45 smithi012 ceph-mon[101180]: osdmap e310: 8 total, 8 up, 7 in 2024-09-14T04:21:45.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:45.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:45.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:45.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:46.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:46.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:46.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:46.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:46.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:46.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:46.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:46.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:21:46.469 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:21:46.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:46.598 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:46.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.726 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:46.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:46.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: pgmap v1155: 109 pgs: 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 32055/149586 objects degraded (21.429%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:21:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:21:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:21:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:21:46.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:46.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:46 smithi012 ceph-mon[101180]: osdmap e311: 8 total, 8 up, 7 in 2024-09-14T04:21:46.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:46.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:46] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-09-14T04:21:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: pgmap v1155: 109 pgs: 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 32055/149586 objects degraded (21.429%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:21:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:21:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:21:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:21:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[107399]: osdmap e311: 8 total, 8 up, 7 in 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: pgmap v1155: 109 pgs: 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 32055/149586 objects degraded (21.429%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:21:46.973 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:46 smithi001 ceph-mon[112165]: osdmap e311: 8 total, 8 up, 7 in 2024-09-14T04:21:46.983 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:46.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:47.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:47.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:47.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:47.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:47.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:47.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:47.473 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:47.580 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:47 smithi001 ceph-mon[107399]: 1.6 scrub starts 2024-09-14T04:21:47.580 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:47 smithi001 ceph-mon[112165]: 1.6 scrub starts 2024-09-14T04:21:47.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:47.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:47.735 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:47.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:47 smithi012 ceph-mon[101180]: 1.6 scrub starts 2024-09-14T04:21:47.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:47.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:47.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:47.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:48.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:48.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:48.136 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:48.239 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:48.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:48.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:48.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:48.491 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:48.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:48.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:48.641 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:48.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:48.765 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:48 smithi012 ceph-mon[101180]: pgmap v1158: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 29742/148698 objects degraded (20.002%); 3159/148698 objects misplaced (2.124%); 44 MiB/s, 700 objects/s recovering 2024-09-14T04:21:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:48 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:21:48.852 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:48 smithi001 ceph-mon[107399]: pgmap v1158: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 29742/148698 objects degraded (20.002%); 3159/148698 objects misplaced (2.124%); 44 MiB/s, 700 objects/s recovering 2024-09-14T04:21:48.852 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:48 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:21:48.853 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:48 smithi001 ceph-mon[112165]: pgmap v1158: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 93 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 29742/148698 objects degraded (20.002%); 3159/148698 objects misplaced (2.124%); 44 MiB/s, 700 objects/s recovering 2024-09-14T04:21:48.853 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:48 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:21:48.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:48.890 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:48.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:48.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:49.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:49.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:49.269 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:49.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:21:49.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:49.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:49.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:49.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:49.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:49.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:49.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:49.713 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:21:49.713 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:21:49.714 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:21:49.714 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:21:49.735 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:49 smithi001 ceph-mon[112165]: osdmap e312: 8 total, 8 up, 7 in 2024-09-14T04:21:49.735 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 29742/148698 objects degraded (20.002%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:49.736 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:49 smithi001 ceph-mon[107399]: osdmap e312: 8 total, 8 up, 7 in 2024-09-14T04:21:49.736 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 29742/148698 objects degraded (20.002%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:49.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:49.775 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:49.822 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:21:49.822 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:21:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:49 smithi012 ceph-mon[101180]: osdmap e312: 8 total, 8 up, 7 in 2024-09-14T04:21:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 29742/148698 objects degraded (20.002%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:49.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:49.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:49.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:50.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.030 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:50.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:50.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:50.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:50.259 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:50.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:50.385 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:50.409 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:50.477 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2024-09-14T04:21:50.477 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:21:50.477 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [1, 3, 4, 5, 6, 7, 0, 2] 2024-09-14T04:21:50.477 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-14T04:21:50.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5 2024-09-14T04:21:50.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:50.538 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:50 smithi012 systemd[1]: Stopping Ceph osd.5 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:21:50.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:50.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:50.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:50.723 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:21:50.757 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:50.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:50.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:50.811 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-mon[101180]: pgmap v1160: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 29738/148665 objects degraded (20.003%); 6212/148665 objects misplaced (4.179%); 35 MiB/s, 568 objects/s recovering 2024-09-14T04:21:50.811 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:21:50.811 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-mon[101180]: osdmap e313: 8 total, 8 up, 7 in 2024-09-14T04:21:50.811 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3009162558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:50.811 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[107399]: pgmap v1160: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 29738/148665 objects degraded (20.003%); 6212/148665 objects misplaced (4.179%); 35 MiB/s, 568 objects/s recovering 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[107399]: osdmap e313: 8 total, 8 up, 7 in 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3009162558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[112165]: pgmap v1160: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 95 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 29738/148665 objects degraded (20.003%); 6212/148665 objects misplaced (4.179%); 35 MiB/s, 568 objects/s recovering 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:21:50.883 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[112165]: osdmap e313: 8 total, 8 up, 7 in 2024-09-14T04:21:50.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3009162558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:21:50.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:50.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:50.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:50.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:50.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:51.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:51.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:51.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[45362]: 2024-09-14T04:21:50.809+0000 7f670bbae640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:21:51.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[45362]: 2024-09-14T04:21:50.809+0000 7f670bbae640 -1 osd.5 313 *** Got signal Terminated *** 2024-09-14T04:21:51.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[45362]: 2024-09-14T04:21:50.809+0000 7f670bbae640 -1 osd.5 313 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:21:51.147 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:51.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:51.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:51.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:51.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:51.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:51.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:51.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:51.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:51.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:51.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:51.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:51.802 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:51 smithi012 ceph-mon[101180]: osd.5 marked itself down and dead 2024-09-14T04:21:51.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:51.928 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:51.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:51.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:51 smithi001 ceph-mon[107399]: osd.5 marked itself down and dead 2024-09-14T04:21:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:51 smithi001 ceph-mon[112165]: osd.5 marked itself down and dead 2024-09-14T04:21:52.030 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:52.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:52.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:52.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:52.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:52.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:52.331 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:52 smithi012 podman[200726]: 2024-09-14 04:21:52.009902045 +0000 UTC m=+1.313309656 container died c828128eebc4104bc9260761725a0c895596231028a0bdd5af3aacbb68bbd4e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b) 2024-09-14T04:21:52.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:52.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:52.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:52.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:52.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:52 smithi012 ceph-mon[101180]: pgmap v1162: 109 pgs: 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 98 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 38 MiB/s, 612 objects/s recovering 2024-09-14T04:21:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:52 smithi012 ceph-mon[101180]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:21:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:52 smithi012 ceph-mon[101180]: osdmap e314: 8 total, 7 up, 7 in 2024-09-14T04:21:52.597 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:52 smithi012 podman[200726]: 2024-09-14 04:21:52.432627186 +0000 UTC m=+1.736034789 container cleanup c828128eebc4104bc9260761725a0c895596231028a0bdd5af3aacbb68bbd4e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:21:52.597 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:52 smithi012 bash[200726]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5 2024-09-14T04:21:52.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:52.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[107399]: pgmap v1162: 109 pgs: 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 98 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 38 MiB/s, 612 objects/s recovering 2024-09-14T04:21:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[107399]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:21:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[107399]: osdmap e314: 8 total, 7 up, 7 in 2024-09-14T04:21:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[112165]: pgmap v1162: 109 pgs: 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 98 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 38 MiB/s, 612 objects/s recovering 2024-09-14T04:21:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[112165]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:21:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:52 smithi001 ceph-mon[112165]: osdmap e314: 8 total, 7 up, 7 in 2024-09-14T04:21:52.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:52.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:52.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:52.888 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:21:52.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:52.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:52.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:53.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:53.074 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:53.089 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:52 smithi012 podman[200833]: 2024-09-14 04:21:52.769294297 +0000 UTC m=+0.719586917 container remove c828128eebc4104bc9260761725a0c895596231028a0bdd5af3aacbb68bbd4e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:21:53.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:53.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:53.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:53.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:53.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:53.346 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:52.991118281 +0000 UTC m=+0.016830772 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:21:53.347 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:53.102111879 +0000 UTC m=+0.127824360 container create 9d00b1ffae7ea07c18a13bea00ec001028196b922f0198ca4761e99c4f14d736 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T04:21:53.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:53.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:53.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:53.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:53.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:53.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:53.811 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:53.838 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:53 smithi012 ceph-mon[101180]: osdmap e315: 8 total, 7 up, 7 in 2024-09-14T04:21:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:53 smithi012 ceph-mon[101180]: 12.c continuing backfill to osd.2 from (300'9587,309'15143] MIN to 309'15143 2024-09-14T04:21:53.847 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:53.513654109 +0000 UTC m=+0.539366599 container init 9d00b1ffae7ea07c18a13bea00ec001028196b922f0198ca4761e99c4f14d736 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:21:53.847 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:53.520417615 +0000 UTC m=+0.546130100 container start 9d00b1ffae7ea07c18a13bea00ec001028196b922f0198ca4761e99c4f14d736 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, ceph=True, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:21:53.847 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:53.622301091 +0000 UTC m=+0.648013628 container attach 9d00b1ffae7ea07c18a13bea00ec001028196b922f0198ca4761e99c4f14d736 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T04:21:53.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:53 smithi001 ceph-mon[112165]: osdmap e315: 8 total, 7 up, 7 in 2024-09-14T04:21:53.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:53 smithi001 ceph-mon[112165]: 12.c continuing backfill to osd.2 from (300'9587,309'15143] MIN to 309'15143 2024-09-14T04:21:53.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:53 smithi001 ceph-mon[107399]: osdmap e315: 8 total, 7 up, 7 in 2024-09-14T04:21:53.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:53 smithi001 ceph-mon[107399]: 12.c continuing backfill to osd.2 from (300'9587,309'15143] MIN to 309'15143 2024-09-14T04:21:53.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:53.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:53.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:53.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:54.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:54.089 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:54.159 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:21:53 smithi012 podman[200991]: 2024-09-14 04:21:53.851169678 +0000 UTC m=+0.876882159 container died 9d00b1ffae7ea07c18a13bea00ec001028196b922f0198ca4761e99c4f14d736 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:21:54.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:54.220 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:54.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:54.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:54.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:54.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:54.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5.service' 2024-09-14T04:21:54.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:54.605 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:21:54.605 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-14T04:21:54.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:54.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:54.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:54 smithi012 ceph-mon[101180]: pgmap v1165: 109 pgs: 18 stale+active+clean, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 18 MiB/s, 294 objects/s recovering 2024-09-14T04:21:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:54 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:21:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 28297/148656 objects degraded (19.035%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:54.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:54.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[107399]: pgmap v1165: 109 pgs: 18 stale+active+clean, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 18 MiB/s, 294 objects/s recovering 2024-09-14T04:21:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:21:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 28297/148656 objects degraded (19.035%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[112165]: pgmap v1165: 109 pgs: 18 stale+active+clean, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28297/148656 objects degraded (19.035%); 6212/148656 objects misplaced (4.179%); 18 MiB/s, 294 objects/s recovering 2024-09-14T04:21:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:21:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 28297/148656 objects degraded (19.035%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:54.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:54.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:55.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:55.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:55.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:55.225 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:55.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:55.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.352 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:55.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:55.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:55.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:55.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:55.711 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:55.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:55.842 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:55.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:55.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:55.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:55.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:56.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.125 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:56.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:56.226 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:56.253 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:56.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:56.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:56.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:56.507 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:56.609 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:56.636 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:56.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:56.765 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:56 smithi012 ceph-mon[101180]: pgmap v1166: 109 pgs: 6 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 18 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 31642/148656 objects degraded (21.285%); 6212/148656 objects misplaced (4.179%); 45 MiB/s, 714 objects/s recovering 2024-09-14T04:21:56.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:56.894 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:56.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:56.968 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:56 smithi001 ceph-mon[107399]: pgmap v1166: 109 pgs: 6 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 18 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 31642/148656 objects degraded (21.285%); 6212/148656 objects misplaced (4.179%); 45 MiB/s, 714 objects/s recovering 2024-09-14T04:21:56.968 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:21:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:21:56] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-14T04:21:56.968 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:56 smithi001 ceph-mon[112165]: pgmap v1166: 109 pgs: 6 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 8 peering, 1 active+undersized+degraded+remapped+backfilling, 18 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 31642/148656 objects degraded (21.285%); 6212/148656 objects misplaced (4.179%); 45 MiB/s, 714 objects/s recovering 2024-09-14T04:21:56.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:57.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:57.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:57.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:57.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:57.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:57.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:57.418 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:57.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:57.551 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:57.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:57.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:57.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:57.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:57.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:57.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:57.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:58.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:21:58.074 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:21:58.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:58.206 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:58.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:21:58.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:21:58.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:58.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:58.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:58.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:58.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:58.692 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:58 smithi001 ceph-mon[107399]: pgmap v1167: 109 pgs: 4 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 29 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 34719/148656 objects degraded (23.355%); 6212/148656 objects misplaced (4.179%); 34 MiB/s, 544 objects/s recovering 2024-09-14T04:21:58.692 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:58 smithi001 ceph-mon[107399]: osdmap e316: 8 total, 7 up, 7 in 2024-09-14T04:21:58.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:58 smithi001 ceph-mon[112165]: pgmap v1167: 109 pgs: 4 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 29 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 34719/148656 objects degraded (23.355%); 6212/148656 objects misplaced (4.179%); 34 MiB/s, 544 objects/s recovering 2024-09-14T04:21:58.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:58 smithi001 ceph-mon[112165]: osdmap e316: 8 total, 7 up, 7 in 2024-09-14T04:21:58.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:58.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:58.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:58 smithi012 ceph-mon[101180]: pgmap v1167: 109 pgs: 4 stale+active+clean, 1 undersized+degraded+remapped+backfilling+peered, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 29 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 34719/148656 objects degraded (23.355%); 6212/148656 objects misplaced (4.179%); 34 MiB/s, 544 objects/s recovering 2024-09-14T04:21:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:58 smithi012 ceph-mon[101180]: osdmap e316: 8 total, 7 up, 7 in 2024-09-14T04:21:58.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:58.963 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:21:58.994 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:58.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:58.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:21:59.096 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:21:59.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:21:59.140 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:21:59.140 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:21:59.167 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:21:59.167 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:21:59.193 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:21:59.193 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:21:59.193 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:21:59.218 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:21:59.219 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:21:59.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:59.246 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:21:59.247 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:21:59.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:21:59.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:21:59.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:21:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:21:59.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:59.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:59.607 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [5] live_osds: [1, 3, 4, 6, 7, 0, 2] 2024-09-14T04:21:59.607 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:21:59.607 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 4, 5, 7, 3] 2024-09-14T04:21:59.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd out 7 2024-09-14T04:21:59.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:59.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:59.786 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:21:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:21:59.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:59 smithi001 ceph-mon[107399]: osdmap e317: 8 total, 7 up, 7 in 2024-09-14T04:21:59.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:21:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 34719/148656 objects degraded (23.355%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:59.842 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:59 smithi001 ceph-mon[112165]: osdmap e317: 8 total, 7 up, 7 in 2024-09-14T04:21:59.842 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:21:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 34719/148656 objects degraded (23.355%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:59 smithi012 ceph-mon[101180]: osdmap e317: 8 total, 7 up, 7 in 2024-09-14T04:21:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:21:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 34719/148656 objects degraded (23.355%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-14T04:21:59.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:21:59.954 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:21:59.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:21:59.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:00.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:00.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:00.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:00.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:00.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:00.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:00.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:00.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:00.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:00.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:00.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:00 smithi001 ceph-mon[112165]: pgmap v1170: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 peering, 32 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 23 op/s; 27126/148329 objects degraded (18.288%); 12815/148329 objects misplaced (8.640%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:22:00.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:00 smithi001 ceph-mon[107399]: pgmap v1170: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 peering, 32 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 23 op/s; 27126/148329 objects degraded (18.288%); 12815/148329 objects misplaced (8.640%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:22:00.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:00.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:00 smithi012 ceph-mon[101180]: pgmap v1170: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 peering, 32 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 23 op/s; 27126/148329 objects degraded (18.288%); 12815/148329 objects misplaced (8.640%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:22:00.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:00.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:00.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:01.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:01.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:01.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:01.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:01.259 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:01.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:01.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:01.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:01.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:01.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:01.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:01.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:01.584 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:22:01.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:01.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:22:01.665 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:22:01.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:01.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3819810401' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:01 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:01 smithi012 ceph-mon[101180]: pgmap v1171: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 32 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 22 op/s; 24821/148275 objects degraded (16.740%); 12805/148275 objects misplaced (8.636%); 72 MiB/s, 1.15k objects/s recovering 2024-09-14T04:22:01.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:01.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:01.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3819810401' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[112165]: pgmap v1171: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 32 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 22 op/s; 24821/148275 objects degraded (16.740%); 12805/148275 objects misplaced (8.636%); 72 MiB/s, 1.15k objects/s recovering 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3819810401' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:22:01.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:01 smithi001 ceph-mon[107399]: pgmap v1171: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 32 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 22 op/s; 24821/148275 objects degraded (16.740%); 12805/148275 objects misplaced (8.636%); 72 MiB/s, 1.15k objects/s recovering 2024-09-14T04:22:02.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:02.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:02.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:02.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:02.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:02.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:02.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:02.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:02.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:02.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:02.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:02.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:02.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:02.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:02 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:02 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:22:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:02 smithi012 ceph-mon[101180]: osdmap e318: 8 total, 7 up, 7 in 2024-09-14T04:22:02.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:02.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:02.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:02.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[107399]: osdmap e318: 8 total, 7 up, 7 in 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:22:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:02 smithi001 ceph-mon[112165]: osdmap e318: 8 total, 7 up, 7 in 2024-09-14T04:22:03.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:03.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:03.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:03.185 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:03.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:03.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:03.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:03.416 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:03.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:03.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:03.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:03.667 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:22:03.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:03.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:03.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:03.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:03 smithi012 ceph-mon[101180]: osdmap e319: 8 total, 7 up, 7 in 2024-09-14T04:22:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:03 smithi012 ceph-mon[101180]: pgmap v1174: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 33 op/s; 24803/148221 objects degraded (16.734%); 12805/148221 objects misplaced (8.639%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:03.859 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:03 smithi001 ceph-mon[112165]: osdmap e319: 8 total, 7 up, 7 in 2024-09-14T04:22:03.859 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:03 smithi001 ceph-mon[112165]: pgmap v1174: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 33 op/s; 24803/148221 objects degraded (16.734%); 12805/148221 objects misplaced (8.639%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:03.860 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:03 smithi001 ceph-mon[107399]: osdmap e319: 8 total, 7 up, 7 in 2024-09-14T04:22:03.860 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:03 smithi001 ceph-mon[107399]: pgmap v1174: 109 pgs: 1 active+remapped+backfilling, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 33 op/s; 24803/148221 objects degraded (16.734%); 12805/148221 objects misplaced (8.639%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:03.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:03.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:03.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:04.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:04.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:04.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:04.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:04.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:04.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:04.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:04.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:04.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:04.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:04.687 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:04.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:04.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:04.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:04.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:04 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:22:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:04 smithi012 ceph-mon[101180]: osdmap e320: 8 total, 7 up, 7 in 2024-09-14T04:22:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 24803/148221 objects degraded (16.734%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:04.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:22:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[112165]: osdmap e320: 8 total, 7 up, 7 in 2024-09-14T04:22:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 24803/148221 objects degraded (16.734%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:04.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:22:04.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[107399]: osdmap e320: 8 total, 7 up, 7 in 2024-09-14T04:22:04.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 24803/148221 objects degraded (16.734%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:04.975 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:04.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:05.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:05.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:05.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:05.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:05.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:05.356 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:05.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:05.484 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:05.586 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:05.611 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:05.713 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:05.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:05.841 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:05.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:22:05.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:05.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:05.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:05.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:06.099 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:06.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:06.129 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:06.130 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:06.130 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[107399]: pgmap v1176: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 23736/146340 objects degraded (16.220%); 7322/146340 objects misplaced (5.003%); 39 MiB/s, 621 objects/s recovering 2024-09-14T04:22:06.130 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:06.130 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:06.130 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:05 smithi001 ceph-mon[112165]: pgmap v1176: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 23736/146340 objects degraded (16.220%); 7322/146340 objects misplaced (5.003%); 39 MiB/s, 621 objects/s recovering 2024-09-14T04:22:06.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:06.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:06.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:06.267 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:22:06.268 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:22:06.268 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:22:06.268 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:22:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:05 smithi012 ceph-mon[101180]: pgmap v1176: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 23736/146340 objects degraded (16.220%); 7322/146340 objects misplaced (5.003%); 39 MiB/s, 621 objects/s recovering 2024-09-14T04:22:06.351 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:06.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:06.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:06.507 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:06.525 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:06.541 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:06.609 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:06.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:06.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:06.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:06.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:06.889 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:06.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:06] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-09-14T04:22:06.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:07.016 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:07.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:07.146 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:07.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:22:07.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:07.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:07.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:07.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:07.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:07.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:07.621 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:07.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:07.746 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:07.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:07.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:07.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:07.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:07.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:08.021 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:08.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:08.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:08.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:08.269 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:08.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:08.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:08.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:08.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:08.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:08 smithi012 ceph-mon[101180]: osdmap e321: 8 total, 7 up, 7 in 2024-09-14T04:22:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:08 smithi012 ceph-mon[101180]: pgmap v1178: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 23619/145812 objects degraded (16.198%); 5003/145812 objects misplaced (3.431%); 40 MiB/s, 634 objects/s recovering 2024-09-14T04:22:08.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:08.652 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:08 smithi001 ceph-mon[107399]: osdmap e321: 8 total, 7 up, 7 in 2024-09-14T04:22:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:08 smithi001 ceph-mon[107399]: pgmap v1178: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 23619/145812 objects degraded (16.198%); 5003/145812 objects misplaced (3.431%); 40 MiB/s, 634 objects/s recovering 2024-09-14T04:22:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:08 smithi001 ceph-mon[112165]: osdmap e321: 8 total, 7 up, 7 in 2024-09-14T04:22:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:08 smithi001 ceph-mon[112165]: pgmap v1178: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 23619/145812 objects degraded (16.198%); 5003/145812 objects misplaced (3.431%); 40 MiB/s, 634 objects/s recovering 2024-09-14T04:22:08.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:08.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:08.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:08.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:08.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:08.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:09.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:09.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:09.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:09.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:09.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:09.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:09.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:09.408 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:09.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:09.423 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:09 smithi001 ceph-mon[107399]: osdmap e322: 8 total, 7 up, 7 in 2024-09-14T04:22:09.423 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 23619/145812 objects degraded (16.198%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:09.424 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:09 smithi001 ceph-mon[112165]: osdmap e322: 8 total, 7 up, 7 in 2024-09-14T04:22:09.424 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 23619/145812 objects degraded (16.198%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:09.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:09.540 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:09 smithi012 ceph-mon[101180]: osdmap e322: 8 total, 7 up, 7 in 2024-09-14T04:22:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 23619/145812 objects degraded (16.198%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:09.642 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:09.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:09.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:09.794 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:09.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:09.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:09.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:09.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:22:09.905+0000 7fbfffcba640 -1 mon.a@0(leader).osd e322 definitely_dead 0 2024-09-14T04:22:10.022 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:10.049 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:10.134 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:10.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:10.180 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:10.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:10.307 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:10.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:10.314 INFO:teuthology.orchestra.run.smithi001.stderr:marked out osd.7. 2024-09-14T04:22:10.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:10.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:10.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:10.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:10 smithi012 ceph-mon[101180]: pgmap v1180: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 23476/144768 objects degraded (16.216%); 4943/144768 objects misplaced (3.414%); 39 MiB/s, 625 objects/s recovering 2024-09-14T04:22:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2973375880' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:10 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:10 smithi012 ceph-mon[101180]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-14T04:22:10.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:10.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[107399]: pgmap v1180: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 23476/144768 objects degraded (16.216%); 4943/144768 objects misplaced (3.414%); 39 MiB/s, 625 objects/s recovering 2024-09-14T04:22:10.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2973375880' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[107399]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-14T04:22:10.682 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[112165]: pgmap v1180: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 33 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 56 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 23476/144768 objects degraded (16.216%); 4943/144768 objects misplaced (3.414%); 39 MiB/s, 625 objects/s recovering 2024-09-14T04:22:10.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2973375880' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-14T04:22:10.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:10 smithi001 ceph-mon[112165]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-14T04:22:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:10.794 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:10.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:10.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:10.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:10.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:11.052 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:11.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:11.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:11.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:11.307 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:11.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:11.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:11.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:11 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-14T04:22:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:11 smithi001 ceph-mon[107399]: osdmap e323: 8 total, 7 up, 6 in 2024-09-14T04:22:11.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:11 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-14T04:22:11.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:11 smithi001 ceph-mon[112165]: osdmap e323: 8 total, 7 up, 6 in 2024-09-14T04:22:11.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:11.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:11.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:11 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-14T04:22:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:11 smithi012 ceph-mon[101180]: osdmap e323: 8 total, 7 up, 6 in 2024-09-14T04:22:11.599 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:11.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:11.727 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:11.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:11.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:11.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:11.984 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:11.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:12.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:12.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:12.203 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:12.217 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:12.250 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:12.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.402 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:12.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: 12.6 continuing backfill to osd.3 from (300'9800,322'15451] 12:69f3bb72:::benchmark_data_smithi001_255951_object110375:head to 322'15451 2024-09-14T04:22:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: 12.0 continuing backfill to osd.2 from (295'8899,322'15108] 12:0ff29228:::benchmark_data_smithi001_255951_object142564:head to 322'15108 2024-09-14T04:22:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: osdmap e324: 8 total, 7 up, 6 in 2024-09-14T04:22:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: 12.7 continuing backfill to osd.2 from (299'9201,322'15336] 12:e2ec9b8c:::benchmark_data_smithi001_255951_object120247:head to 322'15336 2024-09-14T04:22:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: pgmap v1183: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 31 active+undersized, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 23317/144009 objects degraded (16.191%); 4113/144009 objects misplaced (2.856%); 14 MiB/s, 222 objects/s recovering 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[107399]: osdmap e325: 8 total, 7 up, 6 in 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: 12.6 continuing backfill to osd.3 from (300'9800,322'15451] 12:69f3bb72:::benchmark_data_smithi001_255951_object110375:head to 322'15451 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: 12.0 continuing backfill to osd.2 from (295'8899,322'15108] 12:0ff29228:::benchmark_data_smithi001_255951_object142564:head to 322'15108 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: osdmap e324: 8 total, 7 up, 6 in 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: 12.7 continuing backfill to osd.2 from (299'9201,322'15336] 12:e2ec9b8c:::benchmark_data_smithi001_255951_object120247:head to 322'15336 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: pgmap v1183: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 31 active+undersized, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 23317/144009 objects degraded (16.191%); 4113/144009 objects misplaced (2.856%); 14 MiB/s, 222 objects/s recovering 2024-09-14T04:22:12.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:12 smithi001 ceph-mon[112165]: osdmap e325: 8 total, 7 up, 6 in 2024-09-14T04:22:12.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:12.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: 12.6 continuing backfill to osd.3 from (300'9800,322'15451] 12:69f3bb72:::benchmark_data_smithi001_255951_object110375:head to 322'15451 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: 12.0 continuing backfill to osd.2 from (295'8899,322'15108] 12:0ff29228:::benchmark_data_smithi001_255951_object142564:head to 322'15108 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: osdmap e324: 8 total, 7 up, 6 in 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: 12.7 continuing backfill to osd.2 from (299'9201,322'15336] 12:e2ec9b8c:::benchmark_data_smithi001_255951_object120247:head to 322'15336 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: pgmap v1183: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 31 active+undersized, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 23317/144009 objects degraded (16.191%); 4113/144009 objects misplaced (2.856%); 14 MiB/s, 222 objects/s recovering 2024-09-14T04:22:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:12 smithi012 ceph-mon[101180]: osdmap e325: 8 total, 7 up, 6 in 2024-09-14T04:22:12.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:12.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:12.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:12.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:12.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:12.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:12.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:12.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:13.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:13.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:13.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:13.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:13.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:13.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:13.329 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:13.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.458 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:13.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.3 starting backfill to osd.4 from (0'0,0'0] MAX to 322'14959 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.2 starting backfill to osd.1 from (0'0,0'0] MAX to 322'15410 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.f starting backfill to osd.1 from (0'0,0'0] MAX to 322'15017 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 322'15526 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.d continuing backfill to osd.2 from (299'9288,322'15435] 12:b05cfdfc:::benchmark_data_smithi001_255951_object136294:head to 322'15435 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 322'15435 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 322'15157 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:22:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[107399]: 12.e starting backfill to osd.3 from (0'0,0'0] MAX to 322'15189 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.3 starting backfill to osd.4 from (0'0,0'0] MAX to 322'14959 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.2 starting backfill to osd.1 from (0'0,0'0] MAX to 322'15410 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.f starting backfill to osd.1 from (0'0,0'0] MAX to 322'15017 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 322'15526 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.d continuing backfill to osd.2 from (299'9288,322'15435] 12:b05cfdfc:::benchmark_data_smithi001_255951_object136294:head to 322'15435 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 322'15435 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 322'15157 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:22:13.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:13 smithi001 ceph-mon[112165]: 12.e starting backfill to osd.3 from (0'0,0'0] MAX to 322'15189 2024-09-14T04:22:13.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:13.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:13.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:13.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:13.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:13.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.3 starting backfill to osd.4 from (0'0,0'0] MAX to 322'14959 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.2 starting backfill to osd.1 from (0'0,0'0] MAX to 322'15410 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.f starting backfill to osd.1 from (0'0,0'0] MAX to 322'15017 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 322'15526 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.d continuing backfill to osd.2 from (299'9288,322'15435] 12:b05cfdfc:::benchmark_data_smithi001_255951_object136294:head to 322'15435 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 322'15435 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 322'15157 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:22:13.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:13 smithi012 ceph-mon[101180]: 12.e starting backfill to osd.3 from (0'0,0'0] MAX to 322'15189 2024-09-14T04:22:13.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:13.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:13.992 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:13.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:14.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:14.139 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:14.241 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:14.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:14.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:14.420 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:14.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:14.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:14.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:14.687 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[112165]: 1.9 scrub starts 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[112165]: 1.9 scrub ok 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[112165]: osdmap e326: 8 total, 7 up, 6 in 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[112165]: pgmap v1186: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 26 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 19588/143790 objects degraded (13.623%); 4113/143790 objects misplaced (2.860%); 24 MiB/s, 387 objects/s recovering 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 23317/144009 objects degraded (16.191%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[107399]: 1.9 scrub starts 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[107399]: 1.9 scrub ok 2024-09-14T04:22:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[107399]: osdmap e326: 8 total, 7 up, 6 in 2024-09-14T04:22:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[107399]: pgmap v1186: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 26 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 19588/143790 objects degraded (13.623%); 4113/143790 objects misplaced (2.860%); 24 MiB/s, 387 objects/s recovering 2024-09-14T04:22:14.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 23317/144009 objects degraded (16.191%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:14.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:14.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:14 smithi012 ceph-mon[101180]: 1.9 scrub starts 2024-09-14T04:22:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:14 smithi012 ceph-mon[101180]: 1.9 scrub ok 2024-09-14T04:22:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:14 smithi012 ceph-mon[101180]: osdmap e326: 8 total, 7 up, 6 in 2024-09-14T04:22:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:14 smithi012 ceph-mon[101180]: pgmap v1186: 109 pgs: 4 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 26 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized+degraded, 54 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 19588/143790 objects degraded (13.623%); 4113/143790 objects misplaced (2.860%); 24 MiB/s, 387 objects/s recovering 2024-09-14T04:22:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 23317/144009 objects degraded (16.191%), 18 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:14.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:14.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:14.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:14.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:15.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:15.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:15.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:15.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:15.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:15.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:15.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[107399]: osdmap e327: 8 total, 7 up, 6 in 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[107399]: 1.12 scrub starts 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[107399]: 1.12 scrub ok 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[112165]: osdmap e327: 8 total, 7 up, 6 in 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[112165]: 1.12 scrub starts 2024-09-14T04:22:15.401 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:15 smithi001 ceph-mon[112165]: 1.12 scrub ok 2024-09-14T04:22:15.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:15.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:15.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:15.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:15.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:15.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:15.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:15 smithi012 ceph-mon[101180]: osdmap e327: 8 total, 7 up, 6 in 2024-09-14T04:22:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:15 smithi012 ceph-mon[101180]: 1.12 scrub starts 2024-09-14T04:22:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:15 smithi012 ceph-mon[101180]: 1.12 scrub ok 2024-09-14T04:22:15.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:15.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:15.986 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:22:15.986 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:22:15.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:15.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:16.013 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:22:16.014 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:22:16.039 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:22:16.039 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:22:16.063 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:22:16.063 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:22:16.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:16.091 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:22:16.091 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:22:16.091 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:22:16.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:16.119 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:22:16.119 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:22:16.144 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:22:16.145 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:22:16.216 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:16.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:16.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:16.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:16.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:16.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:16.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:16.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:16.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[107399]: pgmap v1188: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 10 active+clean+remapped, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 43 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 18054/142056 objects degraded (12.709%); 23980/142056 objects misplaced (16.881%); 43 MiB/s, 690 objects/s recovering 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[107399]: 1.10 scrub starts 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[107399]: 1.10 scrub ok 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2091900782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[112165]: pgmap v1188: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 10 active+clean+remapped, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 43 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 18054/142056 objects degraded (12.709%); 23980/142056 objects misplaced (16.881%); 43 MiB/s, 690 objects/s recovering 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[112165]: 1.10 scrub starts 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[112165]: 1.10 scrub ok 2024-09-14T04:22:16.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2091900782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:16.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:16.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:16 smithi012 ceph-mon[101180]: pgmap v1188: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 10 active+clean+remapped, 8 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 7 remapped+peering, 1 unknown, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 43 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 18054/142056 objects degraded (12.709%); 23980/142056 objects misplaced (16.881%); 43 MiB/s, 690 objects/s recovering 2024-09-14T04:22:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:16 smithi012 ceph-mon[101180]: 1.10 scrub starts 2024-09-14T04:22:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:16 smithi012 ceph-mon[101180]: 1.10 scrub ok 2024-09-14T04:22:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2091900782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:16.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:16.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:16.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:16.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:16] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-09-14T04:22:17.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:17.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:17.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:17.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:17.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:17.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:17.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:17.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:17.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:17.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:17.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:17.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:17.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:17.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:17 smithi001 ceph-mon[107399]: 1.0 scrub starts 2024-09-14T04:22:17.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:17 smithi001 ceph-mon[107399]: 1.0 scrub ok 2024-09-14T04:22:17.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:17 smithi001 ceph-mon[112165]: 1.0 scrub starts 2024-09-14T04:22:17.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:17 smithi001 ceph-mon[112165]: 1.0 scrub ok 2024-09-14T04:22:17.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:17.806 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:17 smithi012 ceph-mon[101180]: 1.0 scrub starts 2024-09-14T04:22:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:17 smithi012 ceph-mon[101180]: 1.0 scrub ok 2024-09-14T04:22:17.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:17.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:17.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:18.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:18.065 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:18.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:18.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:18.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:18.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:18.323 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:18.394 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:22:18.424 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:18.457 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:18.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:18.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:18.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:18.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: osdmap e328: 8 total, 7 up, 6 in 2024-09-14T04:22:18.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/957509048' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: pgmap v1190: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 4 peering, 1 active+undersized+degraded+remapped+backfilling, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 45 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 20883/141420 objects degraded (14.767%); 32792/141420 objects misplaced (23.188%); 26 MiB/s, 421 objects/s recovering 2024-09-14T04:22:18.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: 1.6 scrub starts 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[107399]: 1.6 scrub ok 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: osdmap e328: 8 total, 7 up, 6 in 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/957509048' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: pgmap v1190: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 4 peering, 1 active+undersized+degraded+remapped+backfilling, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 45 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 20883/141420 objects degraded (14.767%); 32792/141420 objects misplaced (23.188%); 26 MiB/s, 421 objects/s recovering 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: 1.6 scrub starts 2024-09-14T04:22:18.715 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:18 smithi001 ceph-mon[112165]: 1.6 scrub ok 2024-09-14T04:22:18.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:18.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:18.841 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:18.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: osdmap e328: 8 total, 7 up, 6 in 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/957509048' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: pgmap v1190: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 4 peering, 1 active+undersized+degraded+remapped+backfilling, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 45 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 20883/141420 objects degraded (14.767%); 32792/141420 objects misplaced (23.188%); 26 MiB/s, 421 objects/s recovering 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: 1.6 scrub starts 2024-09-14T04:22:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:18 smithi012 ceph-mon[101180]: 1.6 scrub ok 2024-09-14T04:22:18.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:18.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:18.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:19.068 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:19.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:19.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:19.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.230 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:19.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:19.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:19.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:19.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:19.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:19.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:19.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:19.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: osdmap e329: 8 total, 7 up, 6 in 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: 1.e scrub ok 2024-09-14T04:22:19.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 20883/141420 objects degraded (14.767%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: osdmap e329: 8 total, 7 up, 6 in 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: 1.e scrub ok 2024-09-14T04:22:19.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 20883/141420 objects degraded (14.767%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:19.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:19.746 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: osdmap e329: 8 total, 7 up, 6 in 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: 1.e scrub ok 2024-09-14T04:22:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 20883/141420 objects degraded (14.767%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:19.848 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:19.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:19.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:19.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:19.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:20.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:20.049 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [5] live_osds: [1, 3, 4, 6, 7, 0, 2] 2024-09-14T04:22:20.049 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:22:20.050 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2024-09-14T04:22:20.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:22:20.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:20.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:20.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:20.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:20.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:20.389 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:20.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:20.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:20.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:20.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: osdmap e330: 8 total, 7 up, 6 in 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: pgmap v1193: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 19132/140580 objects degraded (13.609%); 30481/140580 objects misplaced (21.682%); 48 MiB/s, 767 objects/s recovering 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: 1.2 scrub starts 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: 1.2 scrub ok 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: osdmap e330: 8 total, 7 up, 6 in 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: pgmap v1193: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 19132/140580 objects degraded (13.609%); 30481/140580 objects misplaced (21.682%); 48 MiB/s, 767 objects/s recovering 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: 1.2 scrub starts 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: 1.2 scrub ok 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:20.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:20.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:20.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: osdmap e330: 8 total, 7 up, 6 in 2024-09-14T04:22:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: pgmap v1193: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 4 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 19132/140580 objects degraded (13.609%); 30481/140580 objects misplaced (21.682%); 48 MiB/s, 767 objects/s recovering 2024-09-14T04:22:20.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: 1.2 scrub starts 2024-09-14T04:22:20.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: 1.2 scrub ok 2024-09-14T04:22:20.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:20.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:20.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:20.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:20.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:21.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:21.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:21.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:21.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:21.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:21.301 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:21.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:21.428 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:21.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:21.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:21.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:21.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:21.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:21.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:21.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:21.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:21.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:21.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:22.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:22.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:22.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:22.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:22.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:22.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:22:22.436 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:22:22.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:22.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:22.522 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:22:22.522 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:22:22.522 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:22 smithi001 ceph-mon[107399]: osdmap e331: 8 total, 7 up, 6 in 2024-09-14T04:22:22.522 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:22 smithi001 ceph-mon[107399]: pgmap v1195: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 19023/139884 objects degraded (13.599%); 28637/139884 objects misplaced (20.472%); 85 MiB/s, 1.36k objects/s recovering 2024-09-14T04:22:22.522 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:22 smithi001 ceph-mon[112165]: osdmap e331: 8 total, 7 up, 6 in 2024-09-14T04:22:22.523 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:22 smithi001 ceph-mon[112165]: pgmap v1195: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 19023/139884 objects degraded (13.599%); 28637/139884 objects misplaced (20.472%); 85 MiB/s, 1.36k objects/s recovering 2024-09-14T04:22:22.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:22.588 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:22.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:22.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:22.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:22.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:22 smithi012 ceph-mon[101180]: osdmap e331: 8 total, 7 up, 6 in 2024-09-14T04:22:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:22 smithi012 ceph-mon[101180]: pgmap v1195: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 6 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 19023/139884 objects degraded (13.599%); 28637/139884 objects misplaced (20.472%); 85 MiB/s, 1.36k objects/s recovering 2024-09-14T04:22:22.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:22.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:22.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:23.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:23.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:23.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:23.228 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:23.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:23.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:23.431 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:22:23.457 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:23.502 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:23.604 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:23.627 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:23 smithi001 ceph-mon[107399]: osdmap e332: 8 total, 7 up, 6 in 2024-09-14T04:22:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:23 smithi001 ceph-mon[112165]: osdmap e332: 8 total, 7 up, 6 in 2024-09-14T04:22:23.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:23.755 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:23.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:23 smithi012 ceph-mon[101180]: osdmap e332: 8 total, 7 up, 6 in 2024-09-14T04:22:23.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:23.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:23.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:23.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:24.008 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:24.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:24.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:24.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:24.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:24.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:24.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:24.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:22:24.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:24.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:24.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:24.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:24.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[107399]: osdmap e333: 8 total, 7 up, 6 in 2024-09-14T04:22:24.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[107399]: pgmap v1198: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 16875/139623 objects degraded (12.086%); 28600/139623 objects misplaced (20.484%); 58 MiB/s, 922 objects/s recovering 2024-09-14T04:22:24.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 19023/139884 objects degraded (13.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:22:24.694 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[112165]: osdmap e333: 8 total, 7 up, 6 in 2024-09-14T04:22:24.694 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[112165]: pgmap v1198: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 16875/139623 objects degraded (12.086%); 28600/139623 objects misplaced (20.484%); 58 MiB/s, 922 objects/s recovering 2024-09-14T04:22:24.694 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 19023/139884 objects degraded (13.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:22:24.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:24.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:24.775 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:24.800 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:24.824 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:24 smithi012 ceph-mon[101180]: osdmap e333: 8 total, 7 up, 6 in 2024-09-14T04:22:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:24 smithi012 ceph-mon[101180]: pgmap v1198: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 5 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 16875/139623 objects degraded (12.086%); 28600/139623 objects misplaced (20.484%); 58 MiB/s, 922 objects/s recovering 2024-09-14T04:22:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 19023/139884 objects degraded (13.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:22:24.864 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:24.888 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:24.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:24.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:24.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:25.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:25.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:25.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:25.246 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:25.271 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:25.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:25.395 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:25.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:25.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:25.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:25.584 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:25 smithi001 ceph-mon[112165]: osdmap e334: 8 total, 7 up, 6 in 2024-09-14T04:22:25.584 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:25 smithi001 ceph-mon[107399]: osdmap e334: 8 total, 7 up, 6 in 2024-09-14T04:22:25.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:25.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:25.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:25.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:25 smithi012 ceph-mon[101180]: osdmap e334: 8 total, 7 up, 6 in 2024-09-14T04:22:25.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:25.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:25.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:26.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:26.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:26.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:26.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:26.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:26.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:26.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:26.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:26.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:26.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:26.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:26.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:26 smithi001 ceph-mon[107399]: pgmap v1200: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 14522/138147 objects degraded (10.512%); 28314/138147 objects misplaced (20.496%); 67 MiB/s, 1.07k objects/s recovering 2024-09-14T04:22:26.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:26 smithi001 ceph-mon[107399]: osdmap e335: 8 total, 7 up, 6 in 2024-09-14T04:22:26.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:26 smithi001 ceph-mon[112165]: pgmap v1200: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 14522/138147 objects degraded (10.512%); 28314/138147 objects misplaced (20.496%); 67 MiB/s, 1.07k objects/s recovering 2024-09-14T04:22:26.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:26 smithi001 ceph-mon[112165]: osdmap e335: 8 total, 7 up, 6 in 2024-09-14T04:22:26.699 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:26.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:22:26.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:26.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:26 smithi012 ceph-mon[101180]: pgmap v1200: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 18 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+remapped+backfill_wait, 22 active+undersized, 4 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 14522/138147 objects degraded (10.512%); 28314/138147 objects misplaced (20.496%); 67 MiB/s, 1.07k objects/s recovering 2024-09-14T04:22:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:26 smithi012 ceph-mon[101180]: osdmap e335: 8 total, 7 up, 6 in 2024-09-14T04:22:26.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:26.959 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:26.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:26.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:26] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:22:27.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:27.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:27.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:27.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:27.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:27.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:27.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:27.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:27.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:27.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:27.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:27.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:27.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:27 smithi001 ceph-mon[107399]: osdmap e336: 8 total, 7 up, 6 in 2024-09-14T04:22:27.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:27 smithi001 ceph-mon[112165]: osdmap e336: 8 total, 7 up, 6 in 2024-09-14T04:22:27.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:27.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:27 smithi012 ceph-mon[101180]: osdmap e336: 8 total, 7 up, 6 in 2024-09-14T04:22:27.936 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:27.964 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:27.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:28.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:28.088 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:28.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:28.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:28.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:28.339 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:28.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:28.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:28.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:28.597 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:28.676 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:28 smithi001 ceph-mon[107399]: pgmap v1203: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 13655/137853 objects degraded (9.905%); 26927/137853 objects misplaced (19.533%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:28.677 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:28 smithi001 ceph-mon[112165]: pgmap v1203: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 13655/137853 objects degraded (9.905%); 26927/137853 objects misplaced (19.533%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:28.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:28.724 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:28.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:28 smithi012 ceph-mon[101180]: pgmap v1203: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 13655/137853 objects degraded (9.905%); 26927/137853 objects misplaced (19.533%); 68 MiB/s, 1.09k objects/s recovering 2024-09-14T04:22:28.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:28.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:28.986 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:28.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:29.088 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:29.117 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:29.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:29.248 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:29.276 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:22:29.278 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:22:29.350 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:29.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 13655/137853 objects degraded (9.905%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:29.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 13655/137853 objects degraded (9.905%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:29.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:29.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:29.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:29.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:29.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:29.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 13655/137853 objects degraded (9.905%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:29.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:29.886 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:29.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:29.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:30.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:30.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:30.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:30.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:30.239 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:30.262 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:30.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:30.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:30.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:30.511 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:30.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:30.536 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:22:30.537 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:22:30.538 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:30 smithi001 ceph-mon[107399]: from='client.45890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:22:30.538 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:30 smithi001 ceph-mon[107399]: pgmap v1204: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 13625/137394 objects degraded (9.917%); 26814/137394 objects misplaced (19.516%); 46 MiB/s, 731 objects/s recovering 2024-09-14T04:22:30.538 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:30 smithi001 ceph-mon[112165]: from='client.45890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:22:30.538 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:30 smithi001 ceph-mon[112165]: pgmap v1204: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 13625/137394 objects degraded (9.917%); 26814/137394 objects misplaced (19.516%); 46 MiB/s, 731 objects/s recovering 2024-09-14T04:22:30.560 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:22:30.560 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:22:30.581 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:22:30.581 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:22:30.597 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:22:30.598 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:22:30.614 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:22:30.614 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:22:30.614 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:22:30.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:30.633 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:22:30.633 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:22:30.648 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:30.652 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:22:30.653 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:22:30.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:30.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:30 smithi012 ceph-mon[101180]: from='client.45890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:22:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:30 smithi012 ceph-mon[101180]: pgmap v1204: 109 pgs: 1 active+recovering+degraded+remapped, 18 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 3 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 13625/137394 objects degraded (9.917%); 26814/137394 objects misplaced (19.516%); 46 MiB/s, 731 objects/s recovering 2024-09-14T04:22:30.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:30.898 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:30.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:30.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:31.000 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:31.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:31.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:31.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:31.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:31.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:31.382 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:31.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:31.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:31.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:22:31.542 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:22:31.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:31.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:31.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:31 smithi001 ceph-mon[107399]: osdmap e337: 8 total, 7 up, 6 in 2024-09-14T04:22:31.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/702912100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:31.714 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:31 smithi001 ceph-mon[112165]: osdmap e337: 8 total, 7 up, 6 in 2024-09-14T04:22:31.714 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/702912100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:31.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:31.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:31 smithi012 ceph-mon[101180]: osdmap e337: 8 total, 7 up, 6 in 2024-09-14T04:22:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/702912100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:31.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:31.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:31.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:32.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:32.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:32.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:32.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:32.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:32.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:32.394 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:32.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:32.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:32.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:32.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:32.671 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:32.717 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:32 smithi001 ceph-mon[107399]: pgmap v1206: 109 pgs: 1 active+recovering+degraded+remapped, 20 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 13297/136134 objects degraded (9.768%); 26556/136134 objects misplaced (19.507%); 20 MiB/s, 316 objects/s recovering 2024-09-14T04:22:32.717 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:32 smithi001 ceph-mon[107399]: osdmap e338: 8 total, 7 up, 6 in 2024-09-14T04:22:32.717 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:32 smithi001 ceph-mon[112165]: pgmap v1206: 109 pgs: 1 active+recovering+degraded+remapped, 20 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 13297/136134 objects degraded (9.768%); 26556/136134 objects misplaced (19.507%); 20 MiB/s, 316 objects/s recovering 2024-09-14T04:22:32.717 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:32 smithi001 ceph-mon[112165]: osdmap e338: 8 total, 7 up, 6 in 2024-09-14T04:22:32.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:32.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:32 smithi012 ceph-mon[101180]: pgmap v1206: 109 pgs: 1 active+recovering+degraded+remapped, 20 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 13297/136134 objects degraded (9.768%); 26556/136134 objects misplaced (19.507%); 20 MiB/s, 316 objects/s recovering 2024-09-14T04:22:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:32 smithi012 ceph-mon[101180]: osdmap e338: 8 total, 7 up, 6 in 2024-09-14T04:22:32.896 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:32.919 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:32.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:32.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:33.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:33.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:33.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:33.176 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:33.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:33.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:33.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:33.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:33.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:33.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:22:33.557 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:33.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:33.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:33.689 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:33.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:33.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:33.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:33.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:33.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:33.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:34.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:34.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:34.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:34.186 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:34.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:34.310 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:34.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:34.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:34.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:34.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:34.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:34.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:34.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:34.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:34.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:34 smithi001 ceph-mon[107399]: pgmap v1208: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 11256/135915 objects degraded (8.282%); 26517/135915 objects misplaced (19.510%); 23 MiB/s, 370 objects/s recovering 2024-09-14T04:22:34.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11256/135915 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:34 smithi001 ceph-mon[112165]: pgmap v1208: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 11256/135915 objects degraded (8.282%); 26517/135915 objects misplaced (19.510%); 23 MiB/s, 370 objects/s recovering 2024-09-14T04:22:34.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11256/135915 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:34.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:34.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:34.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:34 smithi012 ceph-mon[101180]: pgmap v1208: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 11256/135915 objects degraded (8.282%); 26517/135915 objects misplaced (19.510%); 23 MiB/s, 370 objects/s recovering 2024-09-14T04:22:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11256/135915 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:34.895 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111729 11.c 11:38344d7b:::benchmark_data_smithi001_228075_object111728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.25361472499998, 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0573153820000001, 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.896 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111729 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.897 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397460+0000", 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8099999999999998e-07 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397510+0000", 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0049999999999997e-05 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397517+0000", 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9469999999999997e-06 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.397522+0000", 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9030000000000003e-06 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410401+0000", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012879214 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410536+0000", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013432900000000001 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.410682+0000", 2024-09-14T04:22:34.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014610799999999999 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420457+0000", 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097754369999999997 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432471+0000", 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012013348 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438812+0000", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063412800000000004 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438822+0000", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0463e-05 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454678+0000", 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0158558950000001 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454775+0000", 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7027000000000005e-05 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111734 11.c 11:3256a745:::benchmark_data_smithi001_228075_object111733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.246590137, 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0505815119999999, 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.903 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111734 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404484+0000", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404486+0000", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.375e-06 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404540+0000", 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4305000000000003e-05 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404552+0000", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2058999999999999e-05 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404560+0000", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5070000000000003e-06 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420484+0000", 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015924223000000001 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420624+0000", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000139889 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.420770+0000", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014612800000000001 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432477+0000", 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011706658999999999 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438869+0000", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006392516 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438876+0000", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1420000000000004e-06 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.454995+0000", 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161184889999999 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455066+0000", 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1219999999999999e-05 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111737 11.c 11:30a256cb:::benchmark_data_smithi001_228075_object111736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.246089842, 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.072352075, 2024-09-14T04:22:34.909 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111737 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404985+0000", 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.404986+0000", 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5600000000000004e-07 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405008+0000", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2568000000000001e-05 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405023+0000", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4442e-05 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.405028+0000", 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1789999999999999e-06 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427311+0000", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022282885999999998 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427456+0000", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014496299999999999 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.428041+0000", 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00058510400000000003 2024-09-14T04:22:34.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438907+0000", 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010866310000000001 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438913+0000", 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455100+0000", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0161871069999999 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for degraded object", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455507+0000", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040602299999999999 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477138+0000", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021631583999999999 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477337+0000", 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000198717 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111746 11.c 11:38d82d32:::benchmark_data_smithi001_228075_object111745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.23700470099999, 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0634765070000001, 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.916 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111746 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414070+0000", 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414071+0000", 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.519e-06 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414094+0000", 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2546999999999999e-05 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414108+0000", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4425e-05 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.414117+0000", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7499999999999992e-06 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438937+0000", 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024819529 2024-09-14T04:22:34.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.438942+0000", 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1930000000000001e-06 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455547+0000", 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.016605078 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455715+0000", 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001685 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455935+0000", 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000219936 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477381+0000", 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.021446094999999998 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477395+0000", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3925e-05 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477458+0000", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2985999999999996e-05 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477546+0000", 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8023999999999996e-05 2024-09-14T04:22:34.922 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111759 11.c 11:3f216db5:::benchmark_data_smithi001_228075_object111758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.20818216999999, 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0348463530000001, 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.923 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111759 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442892+0000", 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442895+0000", 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4980000000000001e-06 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442917+0000", 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2342999999999999e-05 2024-09-14T04:22:34.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442932+0000", 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4917e-05 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.442940+0000", 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.892e-06 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443001+0000", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1416999999999998e-05 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.443012+0000", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.092e-05 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459584+0000", 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0165711500000001 2024-09-14T04:22:34.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459713+0000", 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012947499999999999 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4", 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459870+0000", 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015698099999999999 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477555+0000", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017684921999999999 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477612+0000", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7117000000000002e-05 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477662+0000", 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9512000000000003e-05 2024-09-14T04:22:34.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.477739+0000", 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7208999999999999e-05 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145641 12.9 12:9c4f5bd4:::benchmark_data_smithi001_255951_object145640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 165.20921907600001, 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4931161319999999, 2024-09-14T04:22:34.930 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145641 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:22:34.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441857+0000", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.615e-06 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441881+0000", 2024-09-14T04:22:34.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4358e-05 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441891+0000", 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1370000000000008e-06 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441899+0000", 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4109999999999996e-06 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.442318+0000", 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00041919 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.442327+0000", 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3850000000000002e-06 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.898231+0000", 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.455904879 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.900067+0000", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0018354840000000001 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.900169+0000", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102407 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.902502+0000", 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023330870000000002 2024-09-14T04:22:34.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.903274+0000", 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00077130899999999997 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934788+0000", 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031514693000000003 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934872+0000", 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3610999999999996e-05 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934972+0000", 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9566000000000002e-05 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243106 12.4 12:24c6f3b1:::benchmark_data_smithi001_255951_object97446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 25.079717949999999, 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.79898594000000001, 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243106 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571358+0000", 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4050000000000001e-06 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571364+0000", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9059999999999999e-06 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571378+0000", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3919e-05 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571384+0000", 2024-09-14T04:22:34.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0950000000000001e-06 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571447+0000", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2761000000000004e-05 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571454+0000", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1380000000000001e-06 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.357976+0000", 2024-09-14T04:22:34.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78652253299999997 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.358107+0000", 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013065900000000001 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.358238+0000", 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013079300000000001 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.362227+0000", 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039890630000000002 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370247+0000", 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080196199999999999 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.943 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370256+0000", 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8689999999999999e-06 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370304+0000", 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7994000000000002e-05 2024-09-14T04:22:34.944 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370343+0000", 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8185000000000001e-05 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243124 12.4 12:26293cfd:::benchmark_data_smithi001_255951_object97464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:07.991335+0000", 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 24.659739550000001, 2024-09-14T04:22:34.945 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.37925590100000001, 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243124 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991335+0000", 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.946 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991335+0000", 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991337+0000", 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.595e-06 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.947 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991342+0000", 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4580000000000001e-06 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991354+0000", 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1985e-05 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991360+0000", 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5949999999999998e-06 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.948 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991420+0000", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0708000000000002e-05 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.991432+0000", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1229e-05 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.362275+0000", 2024-09-14T04:22:34.949 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.370843586 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.362404+0000", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012845399999999999 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.362529+0000", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000124857 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370471+0000", 2024-09-14T04:22:34.950 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079421470000000001 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370527+0000", 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6144999999999999e-05 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370536+0000", 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1910000000000006e-06 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.951 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370568+0000", 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1798999999999998e-05 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370591+0000", 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3152000000000001e-05 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243551 12.4 12:26642039:::benchmark_data_smithi001_255951_object97891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 22.108263665999999, 2024-09-14T04:22:34.952 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4444095940000001, 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243551 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:22:34.953 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542813+0000", 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.809e-06 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.954 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542820+0000", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6129999999999998e-06 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542845+0000", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4627999999999999e-05 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542855+0000", 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.041e-05 2024-09-14T04:22:34.955 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542933+0000", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8009000000000005e-05 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542946+0000", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2379e-05 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317323+0000", 2024-09-14T04:22:34.956 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.77437684399999995 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317337+0000", 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4119e-05 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932714+0000", 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6153777499999999 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.957 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932847+0000", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013261300000000001 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.933426+0000", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00057883999999999995 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952181+0000", 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018755323000000001 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.958 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952240+0000", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8390999999999997e-05 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952638+0000", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00039805999999999999 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.986992+0000", 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.034354518000000001 2024-09-14T04:22:34.959 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987114+0000", 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000121679 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987221+0000", 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106609 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.960 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243553 12.4 12:2f2271ff:::benchmark_data_smithi001_255951_object97893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.991515564, 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.338288382, 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243553 2024-09-14T04:22:34.961 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.962 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659562+0000", 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.492e-06 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659576+0000", 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4008999999999999e-05 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659597+0000", 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1169e-05 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.963 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659607+0000", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0511e-05 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659683+0000", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5479e-05 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659695+0000", 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2529e-05 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.964 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317362+0000", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.65766659800000005 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317369+0000", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3599999999999998e-06 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952702+0000", 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.635333283 2024-09-14T04:22:34.965 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.970997+0000", 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018294331 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971219+0000", 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022208699999999999 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.966 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987267+0000", 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016048271999999999 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987279+0000", 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2177e-05 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987311+0000", 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1829000000000003e-05 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.967 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997605+0000", 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010293771 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997713+0000", 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108224 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.968 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997847+0000", 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013426099999999999 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243556 12.4 12:21983187:::benchmark_data_smithi001_255951_object97896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.746863677, 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0939079889999999, 2024-09-14T04:22:34.969 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243556 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.970 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904213+0000", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3609999999999999e-06 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904221+0000", 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9589999999999995e-06 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.971 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904242+0000", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0871000000000001e-05 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904252+0000", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6479999999999998e-06 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904325+0000", 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3492999999999996e-05 2024-09-14T04:22:34.972 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904338+0000", 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2510000000000001e-05 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317390+0000", 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41305234899999999 2024-09-14T04:22:34.973 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317402+0000", 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1776e-05 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.980836+0000", 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6634340249999999 2024-09-14T04:22:34.974 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.980939+0000", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103546 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.981063+0000", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012321599999999999 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987347+0000", 2024-09-14T04:22:34.975 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062843559999999996 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987385+0000", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8389999999999997e-05 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.995080+0000", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0076942310000000002 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997925+0000", 2024-09-14T04:22:34.976 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028455440000000002 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998016+0000", 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0470000000000006e-05 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998119+0000", 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103244 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.977 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243559 12.4 12:2d42a702:::benchmark_data_smithi001_255951_object97899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 21.602386335999999, 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.950219755, 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243559 2024-09-14T04:22:34.978 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048690+0000", 2024-09-14T04:22:34.979 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9259999999999999e-06 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048697+0000", 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2930000000000003e-06 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048716+0000", 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8921999999999999e-05 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.980 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048724+0000", 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4499999999999998e-06 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048781+0000", 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6925999999999997e-05 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048792+0000", 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1717e-05 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.981 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317428+0000", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26863533699999997 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317446+0000", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7995e-05 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987414+0000", 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.669967846 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.982 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987529+0000", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115388 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987651+0000", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000122014 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.995878+0000", 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0082268069999999992 2024-09-14T04:22:34.983 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997223+0000", 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001345197 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998455+0000", 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012323010000000001 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.984 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998493+0000", 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7530999999999997e-05 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998580+0000", 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7101999999999994e-05 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.985 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998908+0000", 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000328003 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243557 12.f e325/324 12:f12fa30c:::benchmark_data_smithi001_255951_object97897:head v 325'15018, mlcod=0'0)", 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.559353+0000", 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.091721368000002, 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41173359399999998, 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.986 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559353+0000", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559353+0000", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559355+0000", 2024-09-14T04:22:34.987 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.31e-06 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559363+0000", 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7669999999999992e-06 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559377+0000", 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3406e-05 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.988 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559388+0000", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0859999999999999e-05 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559453+0000", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5105999999999996e-05 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.559473+0000", 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0044999999999999e-05 2024-09-14T04:22:34.989 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971039+0000", 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41156666200000003 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971087+0000", 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7438000000000002e-05 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.990 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243563 12.f e325/324 12:f8b22b8d:::benchmark_data_smithi001_255951_object97903:head v 325'15019, mlcod=0'0)", 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.567214+0000", 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.083860986000001, 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.40390222100000001, 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.567214+0000", 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.991 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.567214+0000", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.567215+0000", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3790000000000001e-06 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.567223+0000", 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0180000000000001e-06 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.992 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.601722+0000", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.034499121000000001 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.601732+0000", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6679999999999996e-06 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.943996+0000", 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34226432800000001 2024-09-14T04:22:34.993 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.944015+0000", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9256e-05 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971090+0000", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.027074804000000001 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971116+0000", 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5647e-05 2024-09-14T04:22:34.994 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243570 12.f e325/324 12:f5666424:::benchmark_data_smithi001_255951_object97910:head v 325'15020, mlcod=0'0)", 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.602143+0000", 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.048931670999998, 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.369240448, 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:34.995 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602143+0000", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602143+0000", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602144+0000", 2024-09-14T04:22:34.996 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.06e-07 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602154+0000", 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0584999999999999e-05 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602164+0000", 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6369999999999994e-06 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.997 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602170+0000", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0449999999999997e-06 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.946341+0000", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34417100900000003 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.946358+0000", 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7469000000000002e-05 2024-09-14T04:22:34.998 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971356+0000", 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024997993999999999 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971383+0000", 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7103000000000001e-05 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:34.999 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243573 12.f e325/324 12:fc14b34a:::benchmark_data_smithi001_255951_object97913:head v 325'15021, mlcod=0'0)", 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.602180+0000", 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.048894485000002, 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.36922924200000001, 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602180+0000", 2024-09-14T04:22:35.000 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602180+0000", 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602180+0000", 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9500000000000001e-07 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:35.001 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602183+0000", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6510000000000001e-06 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602188+0000", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3000000000000001e-06 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602198+0000", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.329e-06 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:35.002 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.948547+0000", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34634947700000002 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.948566+0000", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9040999999999999e-05 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971386+0000", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.022819434 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971409+0000", 2024-09-14T04:22:35.003 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3815000000000002e-05 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243567 12.2 e325/324 12:4d08bcc5:::benchmark_data_smithi001_255951_object97907:head v 325'15413, mlcod=0'0)", 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.602274+0000", 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.048800266000001, 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34559349099999997, 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.004 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602274+0000", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602274+0000", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602274+0000", 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7499999999999999e-07 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.005 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602277+0000", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2790000000000001e-06 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602281+0000", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.702e-06 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602286+0000", 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.476e-06 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.006 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.942645+0000", 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34035860200000001 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.942666+0000", 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1242000000000001e-05 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.007 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.947841+0000", 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051756650000000003 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.947868+0000", 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.635e-05 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243571 12.2 e325/324 12:4bd47298:::benchmark_data_smithi001_255951_object97911:head v 325'15414, mlcod=0'0)", 2024-09-14T04:22:35.008 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.602408+0000", 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.048666138000002, 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34794666400000002, 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602408+0000", 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.009 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602408+0000", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602409+0000", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4700000000000001e-07 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602414+0000", 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6160000000000004e-06 2024-09-14T04:22:35.010 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602423+0000", 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2229999999999996e-06 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.602428+0000", 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4569999999999996e-06 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:35.011 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.944736+0000", 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34230748300000002 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.944758+0000", 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2158999999999999e-05 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.950308+0000", 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0055504209999999998 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.012 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.950355+0000", 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6657999999999999e-05 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:243574 12.2 e325/324 12:45c0d6a7:::benchmark_data_smithi001_255951_object97914:head v 325'15415, mlcod=0'0)", 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.603871+0000", 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.047203477, 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.346812122, 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:35.013 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.603871+0000", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.603871+0000", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:35.014 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.603872+0000", 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8800000000000002e-07 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.603877+0000", 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4859999999999997e-06 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932157+0000", 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.32827925800000002 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.015 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932168+0000", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0891e-05 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.947899+0000", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015731961999999999 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.947919+0000", 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9584000000000001e-05 2024-09-14T04:22:35.016 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.950655+0000", 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027361740000000001 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.950683+0000", 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7979000000000001e-05 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.017 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243581 12.d 12:b503f16f:::benchmark_data_smithi001_255951_object97921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:12.649720+0000", 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 20.001354868, 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.63688538400000005, 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243581 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.018 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.649720+0000", 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.649720+0000", 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:35.019 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.649722+0000", 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2560000000000001e-06 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.649732+0000", 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7520000000000008e-06 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932119+0000", 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.28238742500000003 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.020 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932136+0000", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6748999999999998e-05 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.989062+0000", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.056926313999999999 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.999000+0000", 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099373619999999999 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.021 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3,7", 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.999115+0000", 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115622 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.007888+0000", 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087724529999999995 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:35.022 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.007955+0000", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7608999999999996e-05 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.008123+0000", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000167517 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.286194+0000", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.27807103700000002 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:22:35.023 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.286444+0000", 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025022800000000002 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.286605+0000", 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016106000000000001 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:35.024 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:22:35.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:35.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:35.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:35.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:35.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:35.301 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:35.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:35.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:35.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:35.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:35.531 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:35.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:35.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:35.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:35.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:35.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:35.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:35.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:35.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:36.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:36.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:36.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:36.190 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:36.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:36.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:36.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:36.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:36.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:36.575 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:36.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:36.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:36.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:36.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:36 smithi012 ceph-mon[101180]: pgmap v1209: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 11100/134070 objects degraded (8.279%); 26146/134070 objects misplaced (19.502%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:36.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:36.951 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:36.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:36.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:36 smithi001 ceph-mon[107399]: pgmap v1209: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 11100/134070 objects degraded (8.279%); 26146/134070 objects misplaced (19.502%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:36.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:36 smithi001 ceph-mon[112165]: pgmap v1209: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 11100/134070 objects degraded (8.279%); 26146/134070 objects misplaced (19.502%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:36] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-09-14T04:22:37.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:37.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:37.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:37.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:37.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:37.332 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:37.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:37.455 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:37.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:37.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:37.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:37.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:37.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:37.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:37.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:37.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:37.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:38.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:38.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:38.135 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1203,"stamp":"2024-09-14T04:22:27.421692+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3010324898,"num_objects":45951,"num_object_clones":0,"num_object_copies":137853,"num_objects_missing_on_primary":7681,"num_objects_missing":7681,"num_objects_degraded":13655,"num_objects_misplaced":26927,"num_objects_unfound":0,"num_objects_dirty":45951,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":245474,"num_write_kb":9322301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34297,"num_bytes_recovered":2244977809,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89387,"ondisk_log_size":89387,"up":275,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":326,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12444764,"kb_used_data":7696132,"kb_used_omap":34,"kb_used_meta":4748509,"kb_avail":550026148,"statfs":{"total":575970213888,"available":563226775552,"internally_reserved":0,"allocated":7880839168,"data_stored":7867800654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35264,"internal_metadata":4862473792},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-38666240,"num_objects":-590,"num_object_clones":0,"num_object_copies":-1770,"num_objects_missing_on_primary":-53,"num_objects_missing":-53,"num_objects_degraded":-3220,"num_objects_misplaced":-1673,"num_objects_unfound":0,"num_objects_dirty":-590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":590,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4392,"num_bytes_recovered":287834112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.009261"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":478,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.416331+0000","last_change":"2024-09-14T04:21:52.564030+0000","last_active":"2024-09-14T04:22:23.416331+0000","last_peered":"2024-09-14T04:22:23.416331+0000","last_clean":"2024-09-14T04:10:20.286343+0000","last_became_active":"2024-09-14T04:21:52.564030+0000","last_became_peered":"2024-09-14T04:21:52.564030+0000","last_unstale":"2024-09-14T04:22:23.416331+0000","last_undegraded":"2024-09-14T04:22:23.416331+0000","last_fullsized":"2024-09-14T04:21:52.549824+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:57.585118+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:57.585118+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T04:22:52.616976+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":310,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.419605+0000","last_change":"2024-09-14T04:22:12.350843+0000","last_active":"2024-09-14T04:22:23.419605+0000","last_peered":"2024-09-14T04:22:23.419605+0000","last_clean":"2024-09-14T04:22:23.419605+0000","last_became_active":"2024-09-14T04:22:12.350452+0000","last_became_peered":"2024-09-14T04:22:12.350452+0000","last_unstale":"2024-09-14T04:22:23.419605+0000","last_undegraded":"2024-09-14T04:22:23.419605+0000","last_fullsized":"2024-09-14T04:22:23.419605+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:46.566287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":81,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419797+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:22:23.419797+0000","last_peered":"2024-09-14T04:22:23.419797+0000","last_clean":"2024-09-14T04:22:23.419797+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:22:23.419797+0000","last_undegraded":"2024-09-14T04:22:23.419797+0000","last_fullsized":"2024-09-14T04:22:23.419797+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":605,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.415692+0000","last_change":"2024-09-14T04:22:15.602791+0000","last_active":"2024-09-14T04:22:23.415692+0000","last_peered":"2024-09-14T04:22:23.415692+0000","last_clean":"2024-09-14T04:22:23.415692+0000","last_became_active":"2024-09-14T04:22:14.381318+0000","last_became_peered":"2024-09-14T04:22:14.381318+0000","last_unstale":"2024-09-14T04:22:23.415692+0000","last_undegraded":"2024-09-14T04:22:23.415692+0000","last_fullsized":"2024-09-14T04:22:23.415692+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:57.919100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.436346+0000","last_change":"2024-09-14T04:22:12.369717+0000","last_active":"2024-09-14T04:22:26.436346+0000","last_peered":"2024-09-14T04:22:26.436346+0000","last_clean":"2024-09-14T04:22:26.436346+0000","last_became_active":"2024-09-14T04:22:12.369302+0000","last_became_peered":"2024-09-14T04:22:12.369302+0000","last_unstale":"2024-09-14T04:22:26.436346+0000","last_undegraded":"2024-09-14T04:22:26.436346+0000","last_fullsized":"2024-09-14T04:22:26.436346+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:16:35.380406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":270,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416752+0000","last_change":"2024-09-14T04:22:11.335771+0000","last_active":"2024-09-14T04:22:23.416752+0000","last_peered":"2024-09-14T04:22:23.416752+0000","last_clean":"2024-09-14T04:22:23.416752+0000","last_became_active":"2024-09-14T04:22:11.335339+0000","last_became_peered":"2024-09-14T04:22:11.335339+0000","last_unstale":"2024-09-14T04:22:23.416752+0000","last_undegraded":"2024-09-14T04:22:23.416752+0000","last_fullsized":"2024-09-14T04:22:23.416752+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:08.383264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":724,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-14T04:22:25.471843+0000","last_change":"2024-09-14T04:21:52.559619+0000","last_active":"2024-09-14T04:22:25.471843+0000","last_peered":"2024-09-14T04:22:25.471843+0000","last_clean":"2024-09-14T04:21:49.671932+0000","last_became_active":"2024-09-14T04:21:52.559619+0000","last_became_peered":"2024-09-14T04:21:52.559619+0000","last_unstale":"2024-09-14T04:22:25.471843+0000","last_undegraded":"2024-09-14T04:22:25.471843+0000","last_fullsized":"2024-09-14T04:21:52.547869+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:19:15.757626+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":269,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418132+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:22:23.418132+0000","last_peered":"2024-09-14T04:22:23.418132+0000","last_clean":"2024-09-14T04:22:23.418132+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:22:23.418132+0000","last_undegraded":"2024-09-14T04:22:23.418132+0000","last_fullsized":"2024-09-14T04:22:23.418132+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419084+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:22:23.419084+0000","last_peered":"2024-09-14T04:22:23.419084+0000","last_clean":"2024-09-14T04:22:23.419084+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:22:23.419084+0000","last_undegraded":"2024-09-14T04:22:23.419084+0000","last_fullsized":"2024-09-14T04:22:23.419084+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":454,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416462+0000","last_change":"2024-09-14T04:22:14.646545+0000","last_active":"2024-09-14T04:22:23.416462+0000","last_peered":"2024-09-14T04:22:23.416462+0000","last_clean":"2024-09-14T04:22:23.416462+0000","last_became_active":"2024-09-14T04:22:14.380455+0000","last_became_peered":"2024-09-14T04:22:14.380455+0000","last_unstale":"2024-09-14T04:22:23.416462+0000","last_undegraded":"2024-09-14T04:22:23.416462+0000","last_fullsized":"2024-09-14T04:22:23.416462+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:28.045758+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":383,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-14T04:22:26.437283+0000","last_change":"2024-09-14T04:22:11.340614+0000","last_active":"2024-09-14T04:22:26.437283+0000","last_peered":"2024-09-14T04:22:26.437283+0000","last_clean":"2024-09-14T04:22:26.437283+0000","last_became_active":"2024-09-14T04:22:11.340018+0000","last_became_peered":"2024-09-14T04:22:11.340018+0000","last_unstale":"2024-09-14T04:22:26.437283+0000","last_undegraded":"2024-09-14T04:22:26.437283+0000","last_fullsized":"2024-09-14T04:22:26.437283+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:38:06.497044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":317,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:23.415880+0000","last_change":"2024-09-14T04:21:52.560772+0000","last_active":"2024-09-14T04:22:23.415880+0000","last_peered":"2024-09-14T04:22:23.415880+0000","last_clean":"2024-09-14T04:21:49.539008+0000","last_became_active":"2024-09-14T04:21:52.560772+0000","last_became_peered":"2024-09-14T04:21:52.560772+0000","last_unstale":"2024-09-14T04:22:23.415880+0000","last_undegraded":"2024-09-14T04:21:52.551194+0000","last_fullsized":"2024-09-14T04:21:52.551100+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:39:37.997747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":610,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419253+0000","last_change":"2024-09-14T04:21:52.562916+0000","last_active":"2024-09-14T04:22:23.419253+0000","last_peered":"2024-09-14T04:22:23.419253+0000","last_clean":"2024-09-14T04:21:49.544393+0000","last_became_active":"2024-09-14T04:21:52.562916+0000","last_became_peered":"2024-09-14T04:21:52.562916+0000","last_unstale":"2024-09-14T04:22:23.419253+0000","last_undegraded":"2024-09-14T04:22:23.419253+0000","last_fullsized":"2024-09-14T04:21:52.554436+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:47.443322+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.0","version":"334'15226","reported_seq":21427,"reported_epoch":334,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:24.645457+0000","last_change":"2024-09-14T04:22:11.376431+0000","last_active":"2024-09-14T04:22:24.645457+0000","last_peered":"2024-09-14T04:22:24.645457+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:22:11.326199+0000","last_became_peered":"2024-09-14T04:22:11.326199+0000","last_unstale":"2024-09-14T04:22:24.645457+0000","last_undegraded":"2024-09-14T04:22:11.310082+0000","last_fullsized":"2024-09-14T04:22:11.309926+0000","mapping_epoch":323,"log_start":"300'9949","ondisk_log_start":"300'9949","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:25:39.029910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188088320,"num_objects":2870,"num_object_clones":0,"num_object_copies":8610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":2870,"num_objects_unfound":0,"num_objects_dirty":2870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15226,"num_write_kb":579072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3061,"num_bytes_recovered":200605696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":2870}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":35,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.420033+0000","last_change":"2024-09-14T04:22:12.332288+0000","last_active":"2024-09-14T04:22:23.420033+0000","last_peered":"2024-09-14T04:22:23.420033+0000","last_clean":"2024-09-14T04:22:23.420033+0000","last_became_active":"2024-09-14T04:22:12.331742+0000","last_became_peered":"2024-09-14T04:22:12.331742+0000","last_unstale":"2024-09-14T04:22:23.420033+0000","last_undegraded":"2024-09-14T04:22:23.420033+0000","last_fullsized":"2024-09-14T04:22:23.420033+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:12.427733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,7],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":88,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.418861+0000","last_change":"2024-09-14T04:22:12.357530+0000","last_active":"2024-09-14T04:22:23.418861+0000","last_peered":"2024-09-14T04:22:23.418861+0000","last_clean":"2024-09-14T04:22:23.418861+0000","last_became_active":"2024-09-14T04:22:12.357093+0000","last_became_peered":"2024-09-14T04:22:12.357093+0000","last_unstale":"2024-09-14T04:22:23.418861+0000","last_undegraded":"2024-09-14T04:22:23.418861+0000","last_fullsized":"2024-09-14T04:22:23.418861+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:58:38.380489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":759,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.436354+0000","last_change":"2024-09-14T04:22:14.376743+0000","last_active":"2024-09-14T04:22:26.436354+0000","last_peered":"2024-09-14T04:22:26.436354+0000","last_clean":"2024-09-14T04:22:26.436354+0000","last_became_active":"2024-09-14T04:22:14.375475+0000","last_became_peered":"2024-09-14T04:22:14.375475+0000","last_unstale":"2024-09-14T04:22:26.436354+0000","last_undegraded":"2024-09-14T04:22:26.436354+0000","last_fullsized":"2024-09-14T04:22:26.436354+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:45.801032+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"336'15261","reported_seq":17045,"reported_epoch":336,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-09-14T04:22:26.609331+0000","last_change":"2024-09-14T04:22:25.311009+0000","last_active":"2024-09-14T04:22:26.609331+0000","last_peered":"2024-09-14T04:22:26.609331+0000","last_clean":"2024-09-14T04:21:49.544062+0000","last_became_active":"2024-09-14T04:22:12.492536+0000","last_became_peered":"2024-09-14T04:22:12.492536+0000","last_unstale":"2024-09-14T04:22:26.609331+0000","last_undegraded":"2024-09-14T04:22:12.361416+0000","last_fullsized":"2024-09-14T04:22:12.317081+0000","mapping_epoch":324,"log_start":"301'10780","ondisk_log_start":"301'10780","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891712,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2041,"num_objects_misplaced":2867,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15261,"num_write_kb":580096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":828,"num_bytes_recovered":54263808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2867}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":243,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419519+0000","last_change":"2024-09-14T04:22:11.331717+0000","last_active":"2024-09-14T04:22:23.419519+0000","last_peered":"2024-09-14T04:22:23.419519+0000","last_clean":"2024-09-14T04:22:23.419519+0000","last_became_active":"2024-09-14T04:22:11.322388+0000","last_became_peered":"2024-09-14T04:22:11.322388+0000","last_unstale":"2024-09-14T04:22:23.419519+0000","last_undegraded":"2024-09-14T04:22:23.419519+0000","last_fullsized":"2024-09-14T04:22:23.419519+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:55.986305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":340,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418978+0000","last_change":"2024-09-14T04:06:46.336518+0000","last_active":"2024-09-14T04:22:23.418978+0000","last_peered":"2024-09-14T04:22:23.418978+0000","last_clean":"2024-09-14T04:22:23.418978+0000","last_became_active":"2024-09-14T04:06:46.335072+0000","last_became_peered":"2024-09-14T04:06:46.335072+0000","last_unstale":"2024-09-14T04:22:23.418978+0000","last_undegraded":"2024-09-14T04:22:23.418978+0000","last_fullsized":"2024-09-14T04:22:23.418978+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:02.214615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":587,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.436690+0000","last_change":"2024-09-14T04:22:14.375750+0000","last_active":"2024-09-14T04:22:26.436690+0000","last_peered":"2024-09-14T04:22:26.436690+0000","last_clean":"2024-09-14T04:22:26.436690+0000","last_became_active":"2024-09-14T04:22:14.374983+0000","last_became_peered":"2024-09-14T04:22:14.374983+0000","last_unstale":"2024-09-14T04:22:26.436690+0000","last_undegraded":"2024-09-14T04:22:26.436690+0000","last_fullsized":"2024-09-14T04:22:26.436690+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:19:21.330261+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"334'15533","reported_seq":15605,"reported_epoch":334,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:24.520807+0000","last_change":"2024-09-14T04:22:12.598738+0000","last_active":"2024-09-14T04:22:24.520807+0000","last_peered":"2024-09-14T04:22:24.520807+0000","last_clean":"2024-09-14T04:22:10.282961+0000","last_became_active":"2024-09-14T04:22:12.537801+0000","last_became_peered":"2024-09-14T04:22:12.537801+0000","last_unstale":"2024-09-14T04:22:24.520807+0000","last_undegraded":"2024-09-14T04:22:24.520807+0000","last_fullsized":"2024-09-14T04:22:24.520807+0000","mapping_epoch":324,"log_start":"301'10247","ondisk_log_start":"301'10247","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5286,"log_dups_size":0,"ondisk_log_size":5286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:52.053635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191954944,"num_objects":2929,"num_object_clones":0,"num_object_copies":8787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2929,"num_objects_unfound":0,"num_objects_dirty":2929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15533,"num_write_kb":590784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2929}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":26,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.419567+0000","last_change":"2024-09-14T04:22:12.360307+0000","last_active":"2024-09-14T04:22:23.419567+0000","last_peered":"2024-09-14T04:22:23.419567+0000","last_clean":"2024-09-14T04:22:23.419567+0000","last_became_active":"2024-09-14T04:22:12.333542+0000","last_became_peered":"2024-09-14T04:22:12.333542+0000","last_unstale":"2024-09-14T04:22:23.419567+0000","last_undegraded":"2024-09-14T04:22:23.419567+0000","last_fullsized":"2024-09-14T04:22:23.419567+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:22:12.435276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":254,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:25.472033+0000","last_change":"2024-09-14T04:22:12.350522+0000","last_active":"2024-09-14T04:22:25.472033+0000","last_peered":"2024-09-14T04:22:25.472033+0000","last_clean":"2024-09-14T04:22:25.472033+0000","last_became_active":"2024-09-14T04:22:12.350174+0000","last_became_peered":"2024-09-14T04:22:12.350174+0000","last_unstale":"2024-09-14T04:22:25.472033+0000","last_undegraded":"2024-09-14T04:22:25.472033+0000","last_fullsized":"2024-09-14T04:22:25.472033+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:02:48.281588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":517,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418219+0000","last_change":"2024-09-14T04:22:18.659519+0000","last_active":"2024-09-14T04:22:23.418219+0000","last_peered":"2024-09-14T04:22:23.418219+0000","last_clean":"2024-09-14T04:22:23.418219+0000","last_became_active":"2024-09-14T04:21:46.501308+0000","last_became_peered":"2024-09-14T04:21:46.501308+0000","last_unstale":"2024-09-14T04:22:23.418219+0000","last_undegraded":"2024-09-14T04:22:23.418219+0000","last_fullsized":"2024-09-14T04:22:23.418219+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:18.659440+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:18.659440+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:00:19.764709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.3","version":"333'15073","reported_seq":20971,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.809346+0000","last_change":"2024-09-14T04:22:19.437462+0000","last_active":"2024-09-14T04:22:23.809346+0000","last_peered":"2024-09-14T04:22:23.809346+0000","last_clean":"2024-09-14T04:22:23.809346+0000","last_became_active":"2024-09-14T04:22:19.401346+0000","last_became_peered":"2024-09-14T04:22:19.401346+0000","last_unstale":"2024-09-14T04:22:23.809346+0000","last_undegraded":"2024-09-14T04:22:23.809346+0000","last_fullsized":"2024-09-14T04:22:23.809346+0000","mapping_epoch":329,"log_start":"301'10221","ondisk_log_start":"301'10221","created":283,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:46.727443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184614912,"num_objects":2817,"num_object_clones":0,"num_object_copies":8451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15073,"num_write_kb":572480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2884,"num_bytes_recovered":189005824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-14T04:22:26.435790+0000","last_change":"2024-09-14T04:21:52.553595+0000","last_active":"2024-09-14T04:22:26.435790+0000","last_peered":"2024-09-14T04:22:26.435790+0000","last_clean":"2024-09-14T04:21:28.465905+0000","last_became_active":"2024-09-14T04:21:52.553595+0000","last_became_peered":"2024-09-14T04:21:52.553595+0000","last_unstale":"2024-09-14T04:22:26.435790+0000","last_undegraded":"2024-09-14T04:22:26.435790+0000","last_fullsized":"2024-09-14T04:21:52.551315+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:24:28.941772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.417327+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:22:23.417327+0000","last_peered":"2024-09-14T04:22:23.417327+0000","last_clean":"2024-09-14T04:22:23.417327+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:22:23.417327+0000","last_undegraded":"2024-09-14T04:22:23.417327+0000","last_fullsized":"2024-09-14T04:22:23.417327+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"335'15654","reported_seq":27915,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.886422+0000","last_change":"2024-09-14T04:22:24.460713+0000","last_active":"2024-09-14T04:22:25.886422+0000","last_peered":"2024-09-14T04:22:25.886422+0000","last_clean":"2024-09-14T04:22:25.886422+0000","last_became_active":"2024-09-14T04:22:24.424800+0000","last_became_peered":"2024-09-14T04:22:24.424800+0000","last_unstale":"2024-09-14T04:22:25.886422+0000","last_undegraded":"2024-09-14T04:22:25.886422+0000","last_fullsized":"2024-09-14T04:22:25.886422+0000","mapping_epoch":333,"log_start":"295'9105","ondisk_log_start":"295'9105","created":283,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:42:31.922071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192151552,"num_objects":2932,"num_object_clones":0,"num_object_copies":8796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15654,"num_write_kb":594752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6068,"num_bytes_recovered":397672448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":540,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.413815+0000","last_change":"2024-09-14T04:22:13.282799+0000","last_active":"2024-09-14T04:22:23.413815+0000","last_peered":"2024-09-14T04:22:23.413815+0000","last_clean":"2024-09-14T04:22:23.413815+0000","last_became_active":"2024-09-14T04:22:12.389991+0000","last_became_peered":"2024-09-14T04:22:12.389991+0000","last_unstale":"2024-09-14T04:22:23.413815+0000","last_undegraded":"2024-09-14T04:22:23.413815+0000","last_fullsized":"2024-09-14T04:22:23.413815+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:03:42.350687+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":267,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416200+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:22:23.416200+0000","last_peered":"2024-09-14T04:22:23.416200+0000","last_clean":"2024-09-14T04:22:23.416200+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:22:23.416200+0000","last_undegraded":"2024-09-14T04:22:23.416200+0000","last_fullsized":"2024-09-14T04:22:23.416200+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":333,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419517+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:22:23.419517+0000","last_peered":"2024-09-14T04:22:23.419517+0000","last_clean":"2024-09-14T04:22:23.419517+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:22:23.419517+0000","last_undegraded":"2024-09-14T04:22:23.419517+0000","last_fullsized":"2024-09-14T04:22:23.419517+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"333'15357","reported_seq":18326,"reported_epoch":333,"state":"active+recovering+degraded+remapped","last_fresh":"2024-09-14T04:22:24.300096+0000","last_change":"2024-09-14T04:22:13.142124+0000","last_active":"2024-09-14T04:22:24.300096+0000","last_peered":"2024-09-14T04:22:24.300096+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:21:59.209599+0000","last_became_peered":"2024-09-14T04:21:59.209599+0000","last_unstale":"2024-09-14T04:22:24.300096+0000","last_undegraded":"2024-09-14T04:21:58.950900+0000","last_fullsized":"2024-09-14T04:22:24.300096+0000","mapping_epoch":308,"log_start":"299'9138","ondisk_log_start":"299'9138","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":6219,"log_dups_size":0,"ondisk_log_size":6219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189988864,"num_objects":2899,"num_object_clones":0,"num_object_copies":8697,"num_objects_missing_on_primary":7681,"num_objects_missing":7681,"num_objects_degraded":2885,"num_objects_misplaced":2899,"num_objects_unfound":0,"num_objects_dirty":2899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15357,"num_write_kb":584192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1433,"num_bytes_recovered":93454336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,2,6","objects":14},{"shards":"1,6","objects":2885}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":468,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:23.413980+0000","last_change":"2024-09-14T04:21:52.566887+0000","last_active":"2024-09-14T04:22:23.413980+0000","last_peered":"2024-09-14T04:22:23.413980+0000","last_clean":"2024-09-14T04:09:48.216463+0000","last_became_active":"2024-09-14T04:21:52.566887+0000","last_became_peered":"2024-09-14T04:21:52.566887+0000","last_unstale":"2024-09-14T04:22:23.413980+0000","last_undegraded":"2024-09-14T04:21:52.560124+0000","last_fullsized":"2024-09-14T04:21:52.560019+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:18.913123+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":338,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.417019+0000","last_change":"2024-09-14T04:21:52.557415+0000","last_active":"2024-09-14T04:22:23.417019+0000","last_peered":"2024-09-14T04:22:23.417019+0000","last_clean":"2024-09-14T04:21:49.539603+0000","last_became_active":"2024-09-14T04:21:52.557415+0000","last_became_peered":"2024-09-14T04:21:52.557415+0000","last_unstale":"2024-09-14T04:22:23.417019+0000","last_undegraded":"2024-09-14T04:22:23.417019+0000","last_fullsized":"2024-09-14T04:21:52.552428+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:05:21.237137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":143,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419810+0000","last_change":"2024-09-14T04:19:47.446271+0000","last_active":"2024-09-14T04:22:23.419810+0000","last_peered":"2024-09-14T04:22:23.419810+0000","last_clean":"2024-09-14T04:22:23.419810+0000","last_became_active":"2024-09-14T04:19:47.445941+0000","last_became_peered":"2024-09-14T04:19:47.445941+0000","last_unstale":"2024-09-14T04:22:23.419810+0000","last_undegraded":"2024-09-14T04:22:23.419810+0000","last_fullsized":"2024-09-14T04:22:23.419810+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:56.606171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":724,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-14T04:22:25.471419+0000","last_change":"2024-09-14T04:21:52.558591+0000","last_active":"2024-09-14T04:22:25.471419+0000","last_peered":"2024-09-14T04:22:25.471419+0000","last_clean":"2024-09-14T04:21:49.671792+0000","last_became_active":"2024-09-14T04:21:52.558591+0000","last_became_peered":"2024-09-14T04:21:52.558591+0000","last_unstale":"2024-09-14T04:22:25.471419+0000","last_undegraded":"2024-09-14T04:22:25.471419+0000","last_fullsized":"2024-09-14T04:21:52.547265+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:08:04.306951+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"333'15540","reported_seq":19656,"reported_epoch":333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:23.845379+0000","last_change":"2024-09-14T04:22:12.626722+0000","last_active":"2024-09-14T04:22:23.845379+0000","last_peered":"2024-09-14T04:22:23.845379+0000","last_clean":"2024-09-14T04:21:42.482681+0000","last_became_active":"2024-09-14T04:22:11.336980+0000","last_became_peered":"2024-09-14T04:22:11.336980+0000","last_unstale":"2024-09-14T04:22:23.845379+0000","last_undegraded":"2024-09-14T04:22:23.845379+0000","last_fullsized":"2024-09-14T04:22:23.845379+0000","mapping_epoch":323,"log_start":"301'10613","ondisk_log_start":"301'10613","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188219392,"num_objects":2872,"num_object_clones":0,"num_object_copies":8616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3861,"num_objects_unfound":0,"num_objects_dirty":2872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15540,"num_write_kb":589184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1992,"num_bytes_recovered":130547712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,7,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2872}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":153,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418870+0000","last_change":"2024-09-14T04:22:11.326608+0000","last_active":"2024-09-14T04:22:23.418870+0000","last_peered":"2024-09-14T04:22:23.418870+0000","last_clean":"2024-09-14T04:22:23.418870+0000","last_became_active":"2024-09-14T04:22:11.326178+0000","last_became_peered":"2024-09-14T04:22:11.326178+0000","last_unstale":"2024-09-14T04:22:23.418870+0000","last_undegraded":"2024-09-14T04:22:23.418870+0000","last_fullsized":"2024-09-14T04:22:23.418870+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:44:58.183622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":330,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419790+0000","last_change":"2024-09-14T04:21:52.558578+0000","last_active":"2024-09-14T04:22:23.419790+0000","last_peered":"2024-09-14T04:22:23.419790+0000","last_clean":"2024-09-14T04:21:49.544623+0000","last_became_active":"2024-09-14T04:21:52.558578+0000","last_became_peered":"2024-09-14T04:21:52.558578+0000","last_unstale":"2024-09-14T04:22:23.419790+0000","last_undegraded":"2024-09-14T04:22:23.419790+0000","last_fullsized":"2024-09-14T04:21:52.553980+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:18:14.139964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"334'15465","reported_seq":21138,"reported_epoch":334,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-09-14T04:22:25.031304+0000","last_change":"2024-09-14T04:22:21.087145+0000","last_active":"2024-09-14T04:22:25.031304+0000","last_peered":"2024-09-14T04:22:25.031304+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:22:11.328837+0000","last_became_peered":"2024-09-14T04:22:11.328837+0000","last_unstale":"2024-09-14T04:22:25.031304+0000","last_undegraded":"2024-09-14T04:22:11.314080+0000","last_fullsized":"2024-09-14T04:22:11.313209+0000","mapping_epoch":323,"log_start":"299'9201","ondisk_log_start":"299'9201","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:36.767508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191430656,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":180,"num_objects_misplaced":2921,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15465,"num_write_kb":588352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2797,"num_bytes_recovered":183304192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2921}],"blocked_by":[],"up_primary":2,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":406,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.418362+0000","last_change":"2024-09-14T04:21:52.564224+0000","last_active":"2024-09-14T04:22:23.418362+0000","last_peered":"2024-09-14T04:22:23.418362+0000","last_clean":"2024-09-14T03:50:51.989359+0000","last_became_active":"2024-09-14T04:21:52.564224+0000","last_became_peered":"2024-09-14T04:21:52.564224+0000","last_unstale":"2024-09-14T04:22:23.418362+0000","last_undegraded":"2024-09-14T04:22:23.418362+0000","last_fullsized":"2024-09-14T04:21:52.559740+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:10.340449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:10.340449+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T04:23:00.100680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":384,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-14T04:22:26.435139+0000","last_change":"2024-09-14T04:22:11.333375+0000","last_active":"2024-09-14T04:22:26.435139+0000","last_peered":"2024-09-14T04:22:26.435139+0000","last_clean":"2024-09-14T04:22:26.435139+0000","last_became_active":"2024-09-14T04:22:11.331417+0000","last_became_peered":"2024-09-14T04:22:11.331417+0000","last_unstale":"2024-09-14T04:22:26.435139+0000","last_undegraded":"2024-09-14T04:22:26.435139+0000","last_fullsized":"2024-09-14T04:22:26.435139+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:19:32.534341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":510,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.472571+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:22:25.472571+0000","last_peered":"2024-09-14T04:22:25.472571+0000","last_clean":"2024-09-14T04:22:25.472571+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:22:25.472571+0000","last_undegraded":"2024-09-14T04:22:25.472571+0000","last_fullsized":"2024-09-14T04:22:25.472571+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"334'15532","reported_seq":15728,"reported_epoch":335,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:25.471675+0000","last_change":"2024-09-14T04:22:12.988960+0000","last_active":"2024-09-14T04:22:25.471675+0000","last_peered":"2024-09-14T04:22:25.471675+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:22:12.528652+0000","last_became_peered":"2024-09-14T04:22:12.528652+0000","last_unstale":"2024-09-14T04:22:25.471675+0000","last_undegraded":"2024-09-14T04:22:12.356489+0000","last_fullsized":"2024-09-14T04:22:12.312913+0000","mapping_epoch":324,"log_start":"299'9288","ondisk_log_start":"299'9288","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":6244,"log_dups_size":0,"ondisk_log_size":6244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139224,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2768,"num_objects_misplaced":2826,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15532,"num_write_kb":587393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":3604480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":2826}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":605,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.415990+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:22:23.415990+0000","last_peered":"2024-09-14T04:22:23.415990+0000","last_clean":"2024-09-14T04:22:23.415990+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:22:23.415990+0000","last_undegraded":"2024-09-14T04:22:23.415990+0000","last_fullsized":"2024-09-14T04:22:23.415990+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:16.243493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":66,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.419340+0000","last_change":"2024-09-14T04:22:12.515285+0000","last_active":"2024-09-14T04:22:23.419340+0000","last_peered":"2024-09-14T04:22:23.419340+0000","last_clean":"2024-09-14T04:22:23.419340+0000","last_became_active":"2024-09-14T04:22:12.352218+0000","last_became_peered":"2024-09-14T04:22:12.352218+0000","last_unstale":"2024-09-14T04:22:23.419340+0000","last_undegraded":"2024-09-14T04:22:23.419340+0000","last_fullsized":"2024-09-14T04:22:23.419340+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:22.791655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":271,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419024+0000","last_change":"2024-09-14T04:21:52.558288+0000","last_active":"2024-09-14T04:22:23.419024+0000","last_peered":"2024-09-14T04:22:23.419024+0000","last_clean":"2024-09-14T04:21:28.465276+0000","last_became_active":"2024-09-14T04:21:52.558288+0000","last_became_peered":"2024-09-14T04:21:52.558288+0000","last_unstale":"2024-09-14T04:22:23.419024+0000","last_undegraded":"2024-09-14T04:22:23.419024+0000","last_fullsized":"2024-09-14T04:21:52.555406+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:43.675418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":107,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419733+0000","last_change":"2024-09-14T04:22:11.335857+0000","last_active":"2024-09-14T04:22:23.419733+0000","last_peered":"2024-09-14T04:22:23.419733+0000","last_clean":"2024-09-14T04:22:23.419733+0000","last_became_active":"2024-09-14T04:22:11.335556+0000","last_became_peered":"2024-09-14T04:22:11.335556+0000","last_unstale":"2024-09-14T04:22:23.419733+0000","last_undegraded":"2024-09-14T04:22:23.419733+0000","last_fullsized":"2024-09-14T04:22:23.419733+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:09:56.248633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"333'15310","reported_seq":15426,"reported_epoch":333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:24.239334+0000","last_change":"2024-09-14T04:22:12.626722+0000","last_active":"2024-09-14T04:22:24.239334+0000","last_peered":"2024-09-14T04:22:24.239334+0000","last_clean":"2024-09-14T04:22:10.293998+0000","last_became_active":"2024-09-14T04:22:12.477452+0000","last_became_peered":"2024-09-14T04:22:12.477452+0000","last_unstale":"2024-09-14T04:22:24.239334+0000","last_undegraded":"2024-09-14T04:22:24.239334+0000","last_fullsized":"2024-09-14T04:22:24.239334+0000","mapping_epoch":324,"log_start":"301'10451","ondisk_log_start":"301'10451","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189005824,"num_objects":2884,"num_object_clones":0,"num_object_copies":8652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2884,"num_objects_unfound":0,"num_objects_dirty":2884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15310,"num_write_kb":582208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":2884}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":724,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-14T04:22:25.471547+0000","last_change":"2024-09-14T04:21:52.559417+0000","last_active":"2024-09-14T04:22:25.471547+0000","last_peered":"2024-09-14T04:22:25.471547+0000","last_clean":"2024-09-14T04:21:49.671865+0000","last_became_active":"2024-09-14T04:21:52.559417+0000","last_became_peered":"2024-09-14T04:21:52.559417+0000","last_unstale":"2024-09-14T04:22:25.471547+0000","last_undegraded":"2024-09-14T04:22:25.471547+0000","last_fullsized":"2024-09-14T04:21:52.547615+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:28:24.858642+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":211,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471339+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:22:25.471339+0000","last_peered":"2024-09-14T04:22:25.471339+0000","last_clean":"2024-09-14T04:22:25.471339+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:22:25.471339+0000","last_undegraded":"2024-09-14T04:22:25.471339+0000","last_fullsized":"2024-09-14T04:22:25.471339+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":486,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416221+0000","last_change":"2024-09-14T04:11:13.270940+0000","last_active":"2024-09-14T04:22:23.416221+0000","last_peered":"2024-09-14T04:22:23.416221+0000","last_clean":"2024-09-14T04:22:23.416221+0000","last_became_active":"2024-09-14T04:11:13.270559+0000","last_became_peered":"2024-09-14T04:11:13.270559+0000","last_unstale":"2024-09-14T04:22:23.416221+0000","last_undegraded":"2024-09-14T04:22:23.416221+0000","last_fullsized":"2024-09-14T04:22:23.416221+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:10:38.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":274,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418562+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:22:23.418562+0000","last_peered":"2024-09-14T04:22:23.418562+0000","last_clean":"2024-09-14T04:22:23.418562+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:22:23.418562+0000","last_undegraded":"2024-09-14T04:22:23.418562+0000","last_fullsized":"2024-09-14T04:22:23.418562+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"334'15131","reported_seq":15209,"reported_epoch":334,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-14T04:22:24.765882+0000","last_change":"2024-09-14T04:22:13.002556+0000","last_active":"2024-09-14T04:22:24.765882+0000","last_peered":"2024-09-14T04:22:24.765882+0000","last_clean":"2024-09-14T04:21:44.494410+0000","last_became_active":"2024-09-14T04:22:12.529126+0000","last_became_peered":"2024-09-14T04:22:12.529126+0000","last_unstale":"2024-09-14T04:22:24.765882+0000","last_undegraded":"2024-09-14T04:22:12.330847+0000","last_fullsized":"2024-09-14T04:22:12.310599+0000","mapping_epoch":324,"log_start":"300'9854","ondisk_log_start":"300'9854","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:28:30.455134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187760640,"num_objects":2865,"num_object_clones":0,"num_object_copies":8595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2864,"num_objects_misplaced":2865,"num_objects_unfound":0,"num_objects_dirty":2865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15131,"num_write_kb":575872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":2865}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":454,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416918+0000","last_change":"2024-09-14T04:22:19.657807+0000","last_active":"2024-09-14T04:22:23.416918+0000","last_peered":"2024-09-14T04:22:23.416918+0000","last_clean":"2024-09-14T04:22:23.416918+0000","last_became_active":"2024-09-14T04:22:14.381159+0000","last_became_peered":"2024-09-14T04:22:14.381159+0000","last_unstale":"2024-09-14T04:22:23.416918+0000","last_undegraded":"2024-09-14T04:22:23.416918+0000","last_fullsized":"2024-09-14T04:22:23.416918+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:16:32.208435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":480,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.414027+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:22:23.414027+0000","last_peered":"2024-09-14T04:22:23.414027+0000","last_clean":"2024-09-14T04:22:23.414027+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:22:23.414027+0000","last_undegraded":"2024-09-14T04:22:23.414027+0000","last_fullsized":"2024-09-14T04:22:23.414027+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":515,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471515+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:22:25.471515+0000","last_peered":"2024-09-14T04:22:25.471515+0000","last_clean":"2024-09-14T04:22:25.471515+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:22:25.471515+0000","last_undegraded":"2024-09-14T04:22:25.471515+0000","last_fullsized":"2024-09-14T04:22:25.471515+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":353,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.417884+0000","last_change":"2024-09-14T04:22:12.348355+0000","last_active":"2024-09-14T04:22:23.417884+0000","last_peered":"2024-09-14T04:22:23.417884+0000","last_clean":"2024-09-14T04:22:23.417884+0000","last_became_active":"2024-09-14T04:22:12.347992+0000","last_became_peered":"2024-09-14T04:22:12.347992+0000","last_unstale":"2024-09-14T04:22:23.417884+0000","last_undegraded":"2024-09-14T04:22:23.417884+0000","last_fullsized":"2024-09-14T04:22:23.417884+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:40:03.401233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"333'15356","reported_seq":21698,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:24.090582+0000","last_change":"2024-09-14T04:21:58.567605+0000","last_active":"2024-09-14T04:22:24.090582+0000","last_peered":"2024-09-14T04:22:24.090582+0000","last_clean":"2024-09-14T04:18:08.362492+0000","last_became_active":"2024-09-14T04:21:58.567605+0000","last_became_peered":"2024-09-14T04:21:58.567605+0000","last_unstale":"2024-09-14T04:22:24.090582+0000","last_undegraded":"2024-09-14T04:21:58.562423+0000","last_fullsized":"2024-09-14T04:21:58.562300+0000","mapping_epoch":316,"log_start":"301'10514","ondisk_log_start":"301'10514","created":283,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4842,"log_dups_size":0,"ondisk_log_size":4842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:33.426272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189792256,"num_objects":2896,"num_object_clones":0,"num_object_copies":8688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2896,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15356,"num_write_kb":584064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2896}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":478,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.416664+0000","last_change":"2024-09-14T04:21:52.564435+0000","last_active":"2024-09-14T04:22:23.416664+0000","last_peered":"2024-09-14T04:22:23.416664+0000","last_clean":"2024-09-14T04:10:20.286343+0000","last_became_active":"2024-09-14T04:21:52.564435+0000","last_became_peered":"2024-09-14T04:21:52.564435+0000","last_unstale":"2024-09-14T04:22:23.416664+0000","last_undegraded":"2024-09-14T04:22:23.416664+0000","last_fullsized":"2024-09-14T04:21:52.550096+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:20:05.532480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:20:05.532480+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T04:22:53.648278+0000","scrub_duration":0.0016211929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":65,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:25.472217+0000","last_change":"2024-09-14T04:21:52.557211+0000","last_active":"2024-09-14T04:22:25.472217+0000","last_peered":"2024-09-14T04:22:25.472217+0000","last_clean":"2024-09-14T03:41:23.740040+0000","last_became_active":"2024-09-14T04:21:52.557211+0000","last_became_peered":"2024-09-14T04:21:52.557211+0000","last_unstale":"2024-09-14T04:22:25.472217+0000","last_undegraded":"2024-09-14T04:21:52.547791+0000","last_fullsized":"2024-09-14T04:21:52.547680+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:45.057162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419907+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:22:23.419907+0000","last_peered":"2024-09-14T04:22:23.419907+0000","last_clean":"2024-09-14T04:22:23.419907+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:22:23.419907+0000","last_undegraded":"2024-09-14T04:22:23.419907+0000","last_fullsized":"2024-09-14T04:22:23.419907+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":352,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419881+0000","last_change":"2024-09-14T04:21:52.567157+0000","last_active":"2024-09-14T04:22:23.419881+0000","last_peered":"2024-09-14T04:22:23.419881+0000","last_clean":"2024-09-14T04:15:30.401324+0000","last_became_active":"2024-09-14T04:21:52.567157+0000","last_became_peered":"2024-09-14T04:21:52.567157+0000","last_unstale":"2024-09-14T04:22:23.419881+0000","last_undegraded":"2024-09-14T04:22:23.419881+0000","last_fullsized":"2024-09-14T04:21:52.561229+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:19.356182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"333'15619","reported_seq":21763,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.469901+0000","last_change":"2024-09-14T04:22:18.462922+0000","last_active":"2024-09-14T04:22:23.469901+0000","last_peered":"2024-09-14T04:22:23.469901+0000","last_clean":"2024-09-14T04:22:23.469901+0000","last_became_active":"2024-09-14T04:22:18.407144+0000","last_became_peered":"2024-09-14T04:22:18.407144+0000","last_unstale":"2024-09-14T04:22:23.469901+0000","last_undegraded":"2024-09-14T04:22:23.469901+0000","last_fullsized":"2024-09-14T04:22:23.469901+0000","mapping_epoch":328,"log_start":"301'10676","ondisk_log_start":"301'10676","created":283,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:40:21.647706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192741376,"num_objects":2941,"num_object_clones":0,"num_object_copies":8823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15619,"num_write_kb":593920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3011,"num_bytes_recovered":197328896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":587,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.437082+0000","last_change":"2024-09-14T04:22:14.382187+0000","last_active":"2024-09-14T04:22:26.437082+0000","last_peered":"2024-09-14T04:22:26.437082+0000","last_clean":"2024-09-14T04:22:26.437082+0000","last_became_active":"2024-09-14T04:22:14.380187+0000","last_became_peered":"2024-09-14T04:22:14.380187+0000","last_unstale":"2024-09-14T04:22:26.437082+0000","last_undegraded":"2024-09-14T04:22:26.437082+0000","last_fullsized":"2024-09-14T04:22:26.437082+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:00.073630+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":365,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-14T04:22:26.437952+0000","last_change":"2024-09-14T04:22:00.263077+0000","last_active":"2024-09-14T04:22:26.437952+0000","last_peered":"2024-09-14T04:22:26.437952+0000","last_clean":"2024-09-14T04:15:30.402130+0000","last_became_active":"2024-09-14T04:22:00.263077+0000","last_became_peered":"2024-09-14T04:22:00.263077+0000","last_unstale":"2024-09-14T04:22:26.437952+0000","last_undegraded":"2024-09-14T04:22:26.437952+0000","last_fullsized":"2024-09-14T04:21:58.563118+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:24.849154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":572,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471637+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:22:25.471637+0000","last_peered":"2024-09-14T04:22:25.471637+0000","last_clean":"2024-09-14T04:22:25.471637+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:22:25.471637+0000","last_undegraded":"2024-09-14T04:22:25.471637+0000","last_fullsized":"2024-09-14T04:22:25.471637+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":301,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419375+0000","last_change":"2024-09-14T04:21:52.561659+0000","last_active":"2024-09-14T04:22:23.419375+0000","last_peered":"2024-09-14T04:22:23.419375+0000","last_clean":"2024-09-14T04:11:11.240219+0000","last_became_active":"2024-09-14T04:21:52.561659+0000","last_became_peered":"2024-09-14T04:21:52.561659+0000","last_unstale":"2024-09-14T04:22:23.419375+0000","last_undegraded":"2024-09-14T04:22:23.419375+0000","last_fullsized":"2024-09-14T04:21:52.556797+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:22:35.857372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.b","version":"333'15334","reported_seq":21571,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:24.072851+0000","last_change":"2024-09-14T04:21:49.578974+0000","last_active":"2024-09-14T04:22:24.072851+0000","last_peered":"2024-09-14T04:22:24.072851+0000","last_clean":"2024-09-14T04:22:24.072851+0000","last_became_active":"2024-09-14T04:21:49.545043+0000","last_became_peered":"2024-09-14T04:21:49.545043+0000","last_unstale":"2024-09-14T04:22:24.072851+0000","last_undegraded":"2024-09-14T04:22:24.072851+0000","last_fullsized":"2024-09-14T04:22:24.072851+0000","mapping_epoch":312,"log_start":"300'9847","ondisk_log_start":"300'9847","created":283,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:17.749297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186515456,"num_objects":2846,"num_object_clones":0,"num_object_copies":8538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15334,"num_write_kb":581760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3066,"num_bytes_recovered":200933376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":517,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.417735+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:22:23.417735+0000","last_peered":"2024-09-14T04:22:23.417735+0000","last_clean":"2024-09-14T04:22:23.417735+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:22:23.417735+0000","last_undegraded":"2024-09-14T04:22:23.417735+0000","last_fullsized":"2024-09-14T04:22:23.417735+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:34.512632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":511,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-14T04:22:25.471458+0000","last_change":"2024-09-14T04:21:52.565140+0000","last_active":"2024-09-14T04:22:25.471458+0000","last_peered":"2024-09-14T04:22:25.471458+0000","last_clean":"2024-09-14T04:21:49.672314+0000","last_became_active":"2024-09-14T04:21:52.565140+0000","last_became_peered":"2024-09-14T04:21:52.565140+0000","last_unstale":"2024-09-14T04:22:25.471458+0000","last_undegraded":"2024-09-14T04:22:25.471458+0000","last_fullsized":"2024-09-14T04:21:52.548461+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:11.859582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":246,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416409+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:22:23.416409+0000","last_peered":"2024-09-14T04:22:23.416409+0000","last_clean":"2024-09-14T04:22:23.416409+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:22:23.416409+0000","last_undegraded":"2024-09-14T04:22:23.416409+0000","last_fullsized":"2024-09-14T04:22:23.416409+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":273,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.416497+0000","last_change":"2024-09-14T04:22:12.351044+0000","last_active":"2024-09-14T04:22:23.416497+0000","last_peered":"2024-09-14T04:22:23.416497+0000","last_clean":"2024-09-14T04:22:23.416497+0000","last_became_active":"2024-09-14T04:22:12.350662+0000","last_became_peered":"2024-09-14T04:22:12.350662+0000","last_unstale":"2024-09-14T04:22:23.416497+0000","last_undegraded":"2024-09-14T04:22:23.416497+0000","last_fullsized":"2024-09-14T04:22:23.416497+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:19:47.118537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"336'14878","reported_seq":20564,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-14T04:22:26.580683+0000","last_change":"2024-09-14T04:22:22.453840+0000","last_active":"2024-09-14T04:22:26.580683+0000","last_peered":"2024-09-14T04:22:26.580683+0000","last_clean":"2024-09-14T04:22:26.580683+0000","last_became_active":"2024-09-14T04:22:22.422277+0000","last_became_peered":"2024-09-14T04:22:22.422277+0000","last_unstale":"2024-09-14T04:22:26.580683+0000","last_undegraded":"2024-09-14T04:22:26.580683+0000","last_fullsized":"2024-09-14T04:22:26.580683+0000","mapping_epoch":331,"log_start":"301'10351","ondisk_log_start":"301'10351","created":283,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":4527,"log_dups_size":0,"ondisk_log_size":4527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:03.862469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177078272,"num_objects":2702,"num_object_clones":0,"num_object_copies":8106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14878,"num_write_kb":562560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2777,"num_bytes_recovered":181993472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":610,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419258+0000","last_change":"2024-09-14T04:21:52.562935+0000","last_active":"2024-09-14T04:22:23.419258+0000","last_peered":"2024-09-14T04:22:23.419258+0000","last_clean":"2024-09-14T04:21:49.544383+0000","last_became_active":"2024-09-14T04:21:52.562935+0000","last_became_peered":"2024-09-14T04:21:52.562935+0000","last_unstale":"2024-09-14T04:22:23.419258+0000","last_undegraded":"2024-09-14T04:22:23.419258+0000","last_fullsized":"2024-09-14T04:21:52.554797+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:21.566424+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":159,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.417594+0000","last_change":"2024-09-14T04:22:12.406469+0000","last_active":"2024-09-14T04:22:23.417594+0000","last_peered":"2024-09-14T04:22:23.417594+0000","last_clean":"2024-09-14T04:22:23.417594+0000","last_became_active":"2024-09-14T04:22:12.405976+0000","last_became_peered":"2024-09-14T04:22:12.405976+0000","last_unstale":"2024-09-14T04:22:23.417594+0000","last_undegraded":"2024-09-14T04:22:23.417594+0000","last_fullsized":"2024-09-14T04:22:23.417594+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:17.107125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":504,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471998+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:22:25.471998+0000","last_peered":"2024-09-14T04:22:25.471998+0000","last_clean":"2024-09-14T04:22:25.471998+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:22:25.471998+0000","last_undegraded":"2024-09-14T04:22:25.471998+0000","last_fullsized":"2024-09-14T04:22:25.471998+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.415821+0000","last_change":"2024-09-14T04:22:11.333704+0000","last_active":"2024-09-14T04:22:23.415821+0000","last_peered":"2024-09-14T04:22:23.415821+0000","last_clean":"2024-09-14T04:22:23.415821+0000","last_became_active":"2024-09-14T04:22:11.333405+0000","last_became_peered":"2024-09-14T04:22:11.333405+0000","last_unstale":"2024-09-14T04:22:23.415821+0000","last_undegraded":"2024-09-14T04:22:23.415821+0000","last_fullsized":"2024-09-14T04:22:23.415821+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:48.945253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.9","version":"333'15122","reported_seq":21584,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:24.087668+0000","last_change":"2024-09-14T04:22:03.672413+0000","last_active":"2024-09-14T04:22:24.087668+0000","last_peered":"2024-09-14T04:22:24.087668+0000","last_clean":"2024-09-14T04:22:24.087668+0000","last_became_active":"2024-09-14T04:22:03.598315+0000","last_became_peered":"2024-09-14T04:22:03.598315+0000","last_unstale":"2024-09-14T04:22:24.087668+0000","last_undegraded":"2024-09-14T04:22:24.087668+0000","last_fullsized":"2024-09-14T04:22:24.087668+0000","mapping_epoch":319,"log_start":"296'8957","ondisk_log_start":"296'8957","created":283,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:18:08.362492+0000","last_clean_scrub_stamp":"2024-09-14T04:18:08.362492+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:53:50.204242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187301888,"num_objects":2858,"num_object_clones":0,"num_object_copies":8574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15122,"num_write_kb":575360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3168,"num_bytes_recovered":207618048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":754,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.436153+0000","last_change":"2024-09-14T04:22:14.378377+0000","last_active":"2024-09-14T04:22:26.436153+0000","last_peered":"2024-09-14T04:22:26.436153+0000","last_clean":"2024-09-14T04:22:26.436153+0000","last_became_active":"2024-09-14T04:22:14.375099+0000","last_became_peered":"2024-09-14T04:22:14.375099+0000","last_unstale":"2024-09-14T04:22:26.436153+0000","last_undegraded":"2024-09-14T04:22:26.436153+0000","last_fullsized":"2024-09-14T04:22:26.436153+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:56.367026+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":267,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418002+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:22:23.418002+0000","last_peered":"2024-09-14T04:22:23.418002+0000","last_clean":"2024-09-14T04:22:23.418002+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:22:23.418002+0000","last_undegraded":"2024-09-14T04:22:23.418002+0000","last_fullsized":"2024-09-14T04:22:23.418002+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":338,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.420498+0000","last_change":"2024-09-14T04:22:11.340813+0000","last_active":"2024-09-14T04:22:23.420498+0000","last_peered":"2024-09-14T04:22:23.420498+0000","last_clean":"2024-09-14T04:22:23.420498+0000","last_became_active":"2024-09-14T04:22:11.340458+0000","last_became_peered":"2024-09-14T04:22:11.340458+0000","last_unstale":"2024-09-14T04:22:23.420498+0000","last_undegraded":"2024-09-14T04:22:23.420498+0000","last_fullsized":"2024-09-14T04:22:23.420498+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:14:35.981802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":308,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419002+0000","last_change":"2024-09-14T04:21:52.555393+0000","last_active":"2024-09-14T04:22:23.419002+0000","last_peered":"2024-09-14T04:22:23.419002+0000","last_clean":"2024-09-14T04:21:49.543955+0000","last_became_active":"2024-09-14T04:21:52.555393+0000","last_became_peered":"2024-09-14T04:21:52.555393+0000","last_unstale":"2024-09-14T04:22:23.419002+0000","last_undegraded":"2024-09-14T04:22:23.419002+0000","last_fullsized":"2024-09-14T04:21:52.551338+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:04.054321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":345,"reported_epoch":334,"state":"active+undersized","last_fresh":"2024-09-14T04:22:24.418541+0000","last_change":"2024-09-14T04:21:52.562469+0000","last_active":"2024-09-14T04:22:24.418541+0000","last_peered":"2024-09-14T04:22:24.418541+0000","last_clean":"2024-09-14T04:21:49.544162+0000","last_became_active":"2024-09-14T04:21:52.562469+0000","last_became_peered":"2024-09-14T04:21:52.562469+0000","last_unstale":"2024-09-14T04:22:24.418541+0000","last_undegraded":"2024-09-14T04:22:24.418541+0000","last_fullsized":"2024-09-14T04:21:52.554570+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:20:09.729306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":352,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419626+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:22:23.419626+0000","last_peered":"2024-09-14T04:22:23.419626+0000","last_clean":"2024-09-14T04:22:23.419626+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:22:23.419626+0000","last_undegraded":"2024-09-14T04:22:23.419626+0000","last_fullsized":"2024-09-14T04:22:23.419626+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":411,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.418527+0000","last_change":"2024-09-14T04:21:52.566416+0000","last_active":"2024-09-14T04:22:23.418527+0000","last_peered":"2024-09-14T04:22:23.418527+0000","last_clean":"2024-09-14T04:21:49.541437+0000","last_became_active":"2024-09-14T04:21:52.566416+0000","last_became_peered":"2024-09-14T04:21:52.566416+0000","last_unstale":"2024-09-14T04:22:23.418527+0000","last_undegraded":"2024-09-14T04:22:23.418527+0000","last_fullsized":"2024-09-14T04:21:52.559137+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:32.829115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":190,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-14T04:22:26.437799+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:22:26.437799+0000","last_peered":"2024-09-14T04:22:26.437799+0000","last_clean":"2024-09-14T04:22:26.437799+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:22:26.437799+0000","last_undegraded":"2024-09-14T04:22:26.437799+0000","last_fullsized":"2024-09-14T04:22:26.437799+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":754,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:26.435888+0000","last_change":"2024-09-14T04:22:14.378928+0000","last_active":"2024-09-14T04:22:26.435888+0000","last_peered":"2024-09-14T04:22:26.435888+0000","last_clean":"2024-09-14T04:22:26.435888+0000","last_became_active":"2024-09-14T04:22:14.376201+0000","last_became_peered":"2024-09-14T04:22:14.376201+0000","last_unstale":"2024-09-14T04:22:26.435888+0000","last_undegraded":"2024-09-14T04:22:26.435888+0000","last_fullsized":"2024-09-14T04:22:26.435888+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:56.833057+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":335,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:25.472089+0000","last_change":"2024-09-14T04:22:12.402436+0000","last_active":"2024-09-14T04:22:25.472089+0000","last_peered":"2024-09-14T04:22:25.472089+0000","last_clean":"2024-09-14T04:22:25.472089+0000","last_became_active":"2024-09-14T04:22:12.402050+0000","last_became_peered":"2024-09-14T04:22:12.402050+0000","last_unstale":"2024-09-14T04:22:25.472089+0000","last_undegraded":"2024-09-14T04:22:25.472089+0000","last_fullsized":"2024-09-14T04:22:25.472089+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:27:07.134847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2024-09-14T04:22:23.415937+0000","last_change":"2024-09-14T04:22:12.409665+0000","last_active":"2024-09-14T04:22:23.415937+0000","last_peered":"2024-09-14T04:22:23.415937+0000","last_clean":"2024-09-14T04:22:23.415937+0000","last_became_active":"2024-09-14T04:22:12.409294+0000","last_became_peered":"2024-09-14T04:22:12.409294+0000","last_unstale":"2024-09-14T04:22:23.415937+0000","last_undegraded":"2024-09-14T04:22:23.415937+0000","last_fullsized":"2024-09-14T04:22:23.415937+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:14:44.552928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.418063+0000","last_change":"2024-09-14T04:21:52.556285+0000","last_active":"2024-09-14T04:22:23.418063+0000","last_peered":"2024-09-14T04:22:23.418063+0000","last_clean":"2024-09-14T04:21:49.539625+0000","last_became_active":"2024-09-14T04:21:52.556285+0000","last_became_peered":"2024-09-14T04:21:52.556285+0000","last_unstale":"2024-09-14T04:22:23.418063+0000","last_undegraded":"2024-09-14T04:22:23.418063+0000","last_fullsized":"2024-09-14T04:21:52.551846+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:30.102020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":383,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-14T04:22:26.435144+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:22:26.435144+0000","last_peered":"2024-09-14T04:22:26.435144+0000","last_clean":"2024-09-14T04:22:26.435144+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:22:26.435144+0000","last_undegraded":"2024-09-14T04:22:26.435144+0000","last_fullsized":"2024-09-14T04:22:26.435144+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-14T04:22:25.472351+0000","last_change":"2024-09-14T04:21:52.553062+0000","last_active":"2024-09-14T04:22:25.472351+0000","last_peered":"2024-09-14T04:22:25.472351+0000","last_clean":"2024-09-14T03:57:10.202555+0000","last_became_active":"2024-09-14T04:21:52.553062+0000","last_became_peered":"2024-09-14T04:21:52.553062+0000","last_unstale":"2024-09-14T04:22:25.472351+0000","last_undegraded":"2024-09-14T04:22:25.472351+0000","last_fullsized":"2024-09-14T04:21:52.547260+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:14.008079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":274,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416105+0000","last_change":"2024-09-14T04:19:48.462846+0000","last_active":"2024-09-14T04:22:23.416105+0000","last_peered":"2024-09-14T04:22:23.416105+0000","last_clean":"2024-09-14T04:22:23.416105+0000","last_became_active":"2024-09-14T04:19:48.462507+0000","last_became_peered":"2024-09-14T04:19:48.462507+0000","last_unstale":"2024-09-14T04:22:23.416105+0000","last_undegraded":"2024-09-14T04:22:23.416105+0000","last_fullsized":"2024-09-14T04:22:23.416105+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:01:18.706890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":270,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.417918+0000","last_change":"2024-09-14T04:22:11.337374+0000","last_active":"2024-09-14T04:22:23.417918+0000","last_peered":"2024-09-14T04:22:23.417918+0000","last_clean":"2024-09-14T04:22:23.417918+0000","last_became_active":"2024-09-14T04:22:11.336224+0000","last_became_peered":"2024-09-14T04:22:11.336224+0000","last_unstale":"2024-09-14T04:22:23.417918+0000","last_undegraded":"2024-09-14T04:22:23.417918+0000","last_fullsized":"2024-09-14T04:22:23.417918+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:42.780834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":523,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471410+0000","last_change":"2024-09-14T04:11:13.270801+0000","last_active":"2024-09-14T04:22:25.471410+0000","last_peered":"2024-09-14T04:22:25.471410+0000","last_clean":"2024-09-14T04:22:25.471410+0000","last_became_active":"2024-09-14T04:11:13.270374+0000","last_became_peered":"2024-09-14T04:11:13.270374+0000","last_unstale":"2024-09-14T04:22:25.471410+0000","last_undegraded":"2024-09-14T04:22:25.471410+0000","last_fullsized":"2024-09-14T04:22:25.471410+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:53.357679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":333,"state":"active+undersized","last_fresh":"2024-09-14T04:22:23.419418+0000","last_change":"2024-09-14T04:21:52.558830+0000","last_active":"2024-09-14T04:22:23.419418+0000","last_peered":"2024-09-14T04:22:23.419418+0000","last_clean":"2024-09-14T03:56:48.208954+0000","last_became_active":"2024-09-14T04:21:52.558830+0000","last_became_peered":"2024-09-14T04:21:52.558830+0000","last_unstale":"2024-09-14T04:22:23.419418+0000","last_undegraded":"2024-09-14T04:22:23.419418+0000","last_fullsized":"2024-09-14T04:21:52.555388+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.528218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":427,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.417062+0000","last_change":"2024-09-14T04:19:47.450267+0000","last_active":"2024-09-14T04:22:23.417062+0000","last_peered":"2024-09-14T04:22:23.417062+0000","last_clean":"2024-09-14T04:22:23.417062+0000","last_became_active":"2024-09-14T04:19:47.449228+0000","last_became_peered":"2024-09-14T04:19:47.449228+0000","last_unstale":"2024-09-14T04:22:23.417062+0000","last_undegraded":"2024-09-14T04:22:23.417062+0000","last_fullsized":"2024-09-14T04:22:23.417062+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:05:44.565360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":333,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-14T04:22:25.471715+0000","last_change":"2024-09-14T04:16:40.483586+0000","last_active":"2024-09-14T04:22:25.471715+0000","last_peered":"2024-09-14T04:22:25.471715+0000","last_clean":"2024-09-14T04:22:25.471715+0000","last_became_active":"2024-09-14T04:16:40.483231+0000","last_became_peered":"2024-09-14T04:16:40.483231+0000","last_unstale":"2024-09-14T04:22:25.471715+0000","last_undegraded":"2024-09-14T04:22:25.471715+0000","last_fullsized":"2024-09-14T04:22:25.471715+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:15.346443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":280,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.416364+0000","last_change":"2024-09-14T04:22:11.335731+0000","last_active":"2024-09-14T04:22:23.416364+0000","last_peered":"2024-09-14T04:22:23.416364+0000","last_clean":"2024-09-14T04:22:23.416364+0000","last_became_active":"2024-09-14T04:22:11.335383+0000","last_became_peered":"2024-09-14T04:22:11.335383+0000","last_unstale":"2024-09-14T04:22:23.416364+0000","last_undegraded":"2024-09-14T04:22:23.416364+0000","last_fullsized":"2024-09-14T04:22:23.416364+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:16.447058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":148,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419111+0000","last_change":"2024-09-14T04:19:48.463613+0000","last_active":"2024-09-14T04:22:23.419111+0000","last_peered":"2024-09-14T04:22:23.419111+0000","last_clean":"2024-09-14T04:22:23.419111+0000","last_became_active":"2024-09-14T04:19:48.463267+0000","last_became_peered":"2024-09-14T04:19:48.463267+0000","last_unstale":"2024-09-14T04:22:23.419111+0000","last_undegraded":"2024-09-14T04:22:23.419111+0000","last_fullsized":"2024-09-14T04:22:23.419111+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:32.881150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":151,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:23.419307+0000","last_change":"2024-09-14T04:21:52.559786+0000","last_active":"2024-09-14T04:22:23.419307+0000","last_peered":"2024-09-14T04:22:23.419307+0000","last_clean":"2024-09-14T04:02:40.597281+0000","last_became_active":"2024-09-14T04:21:52.559786+0000","last_became_peered":"2024-09-14T04:21:52.559786+0000","last_unstale":"2024-09-14T04:22:23.419307+0000","last_undegraded":"2024-09-14T04:21:52.554808+0000","last_fullsized":"2024-09-14T04:21:52.554707+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:02.662725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419265+0000","last_change":"2024-09-14T04:11:13.271002+0000","last_active":"2024-09-14T04:22:23.419265+0000","last_peered":"2024-09-14T04:22:23.419265+0000","last_clean":"2024-09-14T04:22:23.419265+0000","last_became_active":"2024-09-14T04:11:13.270612+0000","last_became_peered":"2024-09-14T04:11:13.270612+0000","last_unstale":"2024-09-14T04:22:23.419265+0000","last_undegraded":"2024-09-14T04:22:23.419265+0000","last_fullsized":"2024-09-14T04:22:23.419265+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:13:09.759626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":109,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:23.416641+0000","last_change":"2024-09-14T04:21:52.556217+0000","last_active":"2024-09-14T04:22:23.416641+0000","last_peered":"2024-09-14T04:22:23.416641+0000","last_clean":"2024-09-14T03:57:05.958526+0000","last_became_active":"2024-09-14T04:21:52.556217+0000","last_became_peered":"2024-09-14T04:21:52.556217+0000","last_unstale":"2024-09-14T04:22:23.416641+0000","last_undegraded":"2024-09-14T04:21:52.549778+0000","last_fullsized":"2024-09-14T04:21:52.549631+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:21.692088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":427,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.413931+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:22:23.413931+0000","last_peered":"2024-09-14T04:22:23.413931+0000","last_clean":"2024-09-14T04:22:23.413931+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:22:23.413931+0000","last_undegraded":"2024-09-14T04:22:23.413931+0000","last_fullsized":"2024-09-14T04:22:23.413931+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":98,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.419822+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:22:23.419822+0000","last_peered":"2024-09-14T04:22:23.419822+0000","last_clean":"2024-09-14T04:22:23.419822+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:22:23.419822+0000","last_undegraded":"2024-09-14T04:22:23.419822+0000","last_fullsized":"2024-09-14T04:22:23.419822+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":314,"reported_epoch":333,"state":"active+clean","last_fresh":"2024-09-14T04:22:23.418997+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:22:23.418997+0000","last_peered":"2024-09-14T04:22:23.418997+0000","last_clean":"2024-09-14T04:22:23.418997+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:22:23.418997+0000","last_undegraded":"2024-09-14T04:22:23.418997+0000","last_fullsized":"2024-09-14T04:22:23.418997+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":303,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2024-09-14T04:22:23.418245+0000","last_change":"2024-09-14T04:21:52.557932+0000","last_active":"2024-09-14T04:22:23.418245+0000","last_peered":"2024-09-14T04:22:23.418245+0000","last_clean":"2024-09-14T04:21:49.539758+0000","last_became_active":"2024-09-14T04:21:52.557932+0000","last_became_peered":"2024-09-14T04:21:52.557932+0000","last_unstale":"2024-09-14T04:22:23.418245+0000","last_undegraded":"2024-09-14T04:21:52.552229+0000","last_fullsized":"2024-09-14T04:21:52.552135+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:13.971990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3009675288,"num_objects":45925,"num_object_clones":0,"num_object_copies":137775,"num_objects_missing_on_primary":7681,"num_objects_missing":7681,"num_objects_degraded":13645,"num_objects_misplaced":26922,"num_objects_unfound":0,"num_objects_dirty":45925,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":245391,"num_write_kb":9322049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34251,"num_bytes_recovered":2244214784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8779800576,"data_stored":8779792432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86199,"ondisk_log_size":86199,"up":42,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":11648,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":22140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22858,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":82,"acting":86,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":22,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644037,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":751377,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":48,"acting":54,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201782,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463420,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725049,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1800636,"kb_used_data":597616,"kb_used_omap":1,"kb_used_meta":1203006,"kb_avail":91944516,"statfs":{"total":95995035648,"available":94151184384,"internally_reserved":0,"allocated":611958784,"data_stored":609920016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1231878598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986692,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2626588,"kb_used_data":1482256,"kb_used_omap":1,"kb_used_meta":1144318,"kb_avail":91118564,"statfs":{"total":95995035648,"available":93305409536,"internally_reserved":0,"allocated":1517830144,"data_stored":1515627627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1171782086},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748721,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2385520,"kb_used_data":1589920,"kb_used_omap":4,"kb_used_meta":795579,"kb_avail":91359632,"statfs":{"total":95995035648,"available":93552263168,"internally_reserved":0,"allocated":1628078080,"data_stored":1625873206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":814673452},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286942,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2459416,"kb_used_data":2183112,"kb_used_omap":10,"kb_used_meta":276277,"kb_avail":91285736,"statfs":{"total":95995035648,"available":93476593664,"internally_reserved":0,"allocated":2235506688,"data_stored":2233312428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":282908247},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468264,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588312,"kb_used_data":741448,"kb_used_omap":5,"kb_used_meta":846842,"kb_avail":92156840,"statfs":{"total":95995035648,"available":94368604160,"internally_reserved":0,"allocated":759242752,"data_stored":757051036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":867166559},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342379,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584292,"kb_used_data":1101780,"kb_used_omap":10,"kb_used_meta":482485,"kb_avail":92160860,"statfs":{"total":95995035648,"available":94372720640,"internally_reserved":0,"allocated":1128222720,"data_stored":1126016341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":494064850},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":92,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1122107392,"data_stored":1122107392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":753274880,"data_stored":753270808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1279524864,"data_stored":1279524864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1619460096,"data_stored":1619460096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1511260160,"data_stored":1511260160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":606928896,"data_stored":606928896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":377618432,"data_stored":377618432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1509625856,"data_stored":1509621784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:22:38.140 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:22:38.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_7 pg_num 26 2024-09-14T04:22:38.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:38.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:38.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:38.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:38.475 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:38.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:38.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:38.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:38.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:38.752 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:38 smithi012 ceph-mon[101180]: pgmap v1210: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 11046/133377 objects degraded (8.282%); 26035/133377 objects misplaced (19.520%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:38.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:38.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:38.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:38.906 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:38 smithi001 ceph-mon[112165]: pgmap v1210: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 11046/133377 objects degraded (8.282%); 26035/133377 objects misplaced (19.520%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:38.906 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:38 smithi001 ceph-mon[107399]: pgmap v1210: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 11046/133377 objects degraded (8.282%); 26035/133377 objects misplaced (19.520%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:22:38.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:39.005 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:39.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:39.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:39.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:39.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:39.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:39.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:39.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:39.516 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:39.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:39.640 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:39.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:39.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.765 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:39.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11046/133377 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:39.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:39.889 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:39.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11046/133377 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11046/133377 objects degraded (8.282%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:39.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:40.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:40.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:22:40.088 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:40.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:40.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:40.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:40.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:40.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:40.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:40.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:40.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:40.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:40.653 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:40.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:40 smithi001 ceph-mon[107399]: pgmap v1211: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 10970/132396 objects degraded (8.286%); 25843/132396 objects misplaced (19.519%); 15 MiB/s, 232 objects/s recovering 2024-09-14T04:22:40.689 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:40 smithi001 ceph-mon[112165]: pgmap v1211: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 10970/132396 objects degraded (8.286%); 25843/132396 objects misplaced (19.519%); 15 MiB/s, 232 objects/s recovering 2024-09-14T04:22:40.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:40.771 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:40.785 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:40.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:40 smithi012 ceph-mon[101180]: pgmap v1211: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 10970/132396 objects degraded (8.286%); 25843/132396 objects misplaced (19.519%); 15 MiB/s, 232 objects/s recovering 2024-09-14T04:22:40.903 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:40.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:40.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:41.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:41.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:41.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:41.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:41.283 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:41.310 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:41.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:41.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:41.502 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:22:41.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:41.542 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3212438523' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.542 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.543 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3212438523' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.543 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:41.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:41.696 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:41.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:41.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3212438523' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:22:41.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:41.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:41.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:42.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:42.087 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:42.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:42.212 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:42.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:42.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:42.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:42.461 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:42.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:42.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:42.660 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[107399]: pgmap v1212: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 10762/130200 objects degraded (8.266%); 25406/130200 objects misplaced (19.513%); 13 MiB/s, 213 objects/s recovering 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[107399]: osdmap e339: 8 total, 7 up, 6 in 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[112165]: pgmap v1212: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 10762/130200 objects degraded (8.266%); 25406/130200 objects misplaced (19.513%); 13 MiB/s, 213 objects/s recovering 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:22:42.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:42 smithi001 ceph-mon[112165]: osdmap e339: 8 total, 7 up, 6 in 2024-09-14T04:22:42.687 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:42.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:42.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:42.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:42 smithi012 ceph-mon[101180]: pgmap v1212: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 10762/130200 objects degraded (8.266%); 25406/130200 objects misplaced (19.513%); 13 MiB/s, 213 objects/s recovering 2024-09-14T04:22:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:42 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:22:42.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:42 smithi012 ceph-mon[101180]: osdmap e339: 8 total, 7 up, 6 in 2024-09-14T04:22:42.939 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:42.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:42.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:43.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:43.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:43.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:43.216 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:43.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:43.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:43.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:43.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:43.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:43.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:43.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:43.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:43.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:43.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:43.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:43.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:43.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:44.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:44.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:44.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:44.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:44.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:44.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:44.459 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:44.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:44.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:44.610 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:44.712 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:44 smithi001 ceph-mon[112165]: pgmap v1214: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 10762/129789 objects degraded (8.292%); 25339/129789 objects misplaced (19.523%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:22:44.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10762/129789 objects degraded (8.292%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:44.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:44 smithi001 ceph-mon[107399]: pgmap v1214: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 10762/129789 objects degraded (8.292%); 25339/129789 objects misplaced (19.523%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:22:44.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10762/129789 objects degraded (8.292%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:44.735 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:44.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:44.855 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:44 smithi012 ceph-mon[101180]: pgmap v1214: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 10762/129789 objects degraded (8.292%); 25339/129789 objects misplaced (19.523%); 710 KiB/s, 11 objects/s recovering 2024-09-14T04:22:44.855 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10762/129789 objects degraded (8.292%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:44.876 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:44.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:44.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:45.007 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:45.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:45.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:45.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:45.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:45.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:45.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.385 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:45.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:45.486 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:45.497 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 12 pg_num to 26 2024-09-14T04:22:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:45.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:45.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:45.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:22:45.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:22:45.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:45.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:22:45.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:22:45.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:45.935 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:45.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:46.023 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:22:46.024 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:22:46.024 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.024 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.45647.0:249229 12.6 e340/323 12:69a54dea:::benchmark_data_smithi001_255951_object103569:head v 340'15792, mlcod=340'15790)", 2024-09-14T04:22:46.024 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:46.020396+0000", 2024-09-14T04:22:46.024 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.001024575, 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00105529, 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020396+0000", 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.025 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020396+0000", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020397+0000", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2410000000000001e-06 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020408+0000", 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0801e-05 2024-09-14T04:22:46.026 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020426+0000", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8017999999999999e-05 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020435+0000", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9120000000000004e-06 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020510+0000", 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4731e-05 2024-09-14T04:22:46.027 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:46.020539+0000", 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8980999999999999e-05 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 1 2024-09-14T04:22:46.028 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:22:46.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:46.067 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:46.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:46.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:46.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:46.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:46.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:46.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:46.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:46.571 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:46.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:46.698 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:46.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[107399]: pgmap v1215: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 10596/127851 objects degraded (8.288%); 24962/127851 objects misplaced (19.524%); 697 KiB/s, 10 objects/s recovering 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[107399]: osdmap e340: 8 total, 7 up, 6 in 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[112165]: pgmap v1215: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 10596/127851 objects degraded (8.288%); 24962/127851 objects misplaced (19.524%); 697 KiB/s, 10 objects/s recovering 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[112165]: osdmap e340: 8 total, 7 up, 6 in 2024-09-14T04:22:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:22:46.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:46.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:46 smithi012 ceph-mon[101180]: pgmap v1215: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 10596/127851 objects degraded (8.288%); 24962/127851 objects misplaced (19.524%); 697 KiB/s, 10 objects/s recovering 2024-09-14T04:22:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2668519857' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:22:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:46 smithi012 ceph-mon[101180]: osdmap e340: 8 total, 7 up, 6 in 2024-09-14T04:22:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:22:46.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:46.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:46.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:46] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-09-14T04:22:47.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:47.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:47.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:47.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:47.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:47.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:47.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:47.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:47.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:47.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.635 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:47.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:47.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:47.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:47.767 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:22:47.807 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:22:47.808 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:22:47.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:47.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:47.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:47.986 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:48.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:48.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:48.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:48.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:48.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:48.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:48.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:48.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:48.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:48.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:48.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:48.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:48.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:48.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:48 smithi012 ceph-mon[101180]: pgmap v1217: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 10536/127053 objects degraded (8.293%); 24827/127053 objects misplaced (19.541%); 872 KiB/s, 13 objects/s recovering 2024-09-14T04:22:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:22:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:48 smithi012 ceph-mon[101180]: osdmap e341: 8 total, 7 up, 6 in 2024-09-14T04:22:48.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:48.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:48.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[107399]: pgmap v1217: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 10536/127053 objects degraded (8.293%); 24827/127053 objects misplaced (19.541%); 872 KiB/s, 13 objects/s recovering 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[107399]: osdmap e341: 8 total, 7 up, 6 in 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[112165]: pgmap v1217: 109 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 53 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 10536/127053 objects degraded (8.293%); 24827/127053 objects misplaced (19.541%); 872 KiB/s, 13 objects/s recovering 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:22:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:48 smithi001 ceph-mon[112165]: osdmap e341: 8 total, 7 up, 6 in 2024-09-14T04:22:49.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:49.052 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:49.117 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:22:49.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:49.180 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:49.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:49.315 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:49.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:49.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:49.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:49.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:49.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:49.682 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:22:49.682 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: osdmap e342: 8 total, 7 up, 6 in 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.6 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.2 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.9 scrub starts 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10536/127053 objects degraded (8.293%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:49.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: osdmap e343: 8 total, 7 up, 6 in 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.5 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.4 scrub starts 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[112165]: 12.8 scrub starts 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: osdmap e342: 8 total, 7 up, 6 in 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.6 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.2 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.9 scrub starts 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10536/127053 objects degraded (8.293%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:49.684 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: osdmap e343: 8 total, 7 up, 6 in 2024-09-14T04:22:49.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.5 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:49.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.4 scrub starts 2024-09-14T04:22:49.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:49 smithi001 ceph-mon[107399]: 12.8 scrub starts 2024-09-14T04:22:49.694 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:49.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:49.704 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:22:49.704 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:22:49.740 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:49.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:49.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: osdmap e342: 8 total, 7 up, 6 in 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.6 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.2 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.9 scrub starts 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10536/127053 objects degraded (8.293%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:22:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: osdmap e343: 8 total, 7 up, 6 in 2024-09-14T04:22:49.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.5 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:49.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.4 scrub starts 2024-09-14T04:22:49.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:49 smithi012 ceph-mon[101180]: 12.8 scrub starts 2024-09-14T04:22:49.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:49.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:49.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:49.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.082 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.316 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.341 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.468 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:50.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:50.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: 12.3 scrub starts 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: pgmap v1221: 119 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 1 remapped+peering, 8 unknown, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 10501/126576 objects degraded (8.296%); 22961/126576 objects misplaced (18.140%) 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4242151272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:50.834 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: 12.7 scrub starts 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: 12.12 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: osdmap e344: 8 total, 7 up, 6 in 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: 12.14 scrub starts 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[112165]: 12.1 scrub starts 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: 12.3 scrub starts 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: pgmap v1221: 119 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 1 remapped+peering, 8 unknown, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 10501/126576 objects degraded (8.296%); 22961/126576 objects misplaced (18.140%) 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4242151272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:50.836 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: 12.7 scrub starts 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: 12.12 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: osdmap e344: 8 total, 7 up, 6 in 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: 12.14 scrub starts 2024-09-14T04:22:50.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:50 smithi001 ceph-mon[107399]: 12.1 scrub starts 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: 12.3 scrub starts 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: pgmap v1221: 119 pgs: 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 1 remapped+peering, 8 unknown, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 10501/126576 objects degraded (8.296%); 22961/126576 objects misplaced (18.140%) 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4242151272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: 12.7 scrub starts 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: 12.12 continuing backfill to osd.1 from (301'10447,340'15795] MIN to 340'15795 2024-09-14T04:22:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: osdmap e344: 8 total, 7 up, 6 in 2024-09-14T04:22:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: 12.14 scrub starts 2024-09-14T04:22:50.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:50 smithi012 ceph-mon[101180]: 12.1 scrub starts 2024-09-14T04:22:50.848 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:50.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:50.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:50.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:51.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:51.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:51.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:51.225 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:51.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.352 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:51.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:51.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:51.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:51.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:51.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:51.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[112165]: 12.16 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[112165]: 12.13 scrub starts 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[112165]: 12.18 scrub starts 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[107399]: 12.16 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[107399]: 12.13 scrub starts 2024-09-14T04:22:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[107399]: 12.18 scrub starts 2024-09-14T04:22:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:51.802 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:22:51.802 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:22:51.834 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:22:51.835 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:22:51.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:51 smithi012 ceph-mon[101180]: 12.16 continuing backfill to osd.3 from (301'10913,340'15807] MIN to 340'15807 2024-09-14T04:22:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:51 smithi012 ceph-mon[101180]: 12.13 scrub starts 2024-09-14T04:22:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:51 smithi012 ceph-mon[101180]: 12.18 scrub starts 2024-09-14T04:22:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:22:51.864 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:51.867 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:22:51.868 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:22:51.868 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:22:51.889 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:22:51.890 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:22:51.909 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:22:51.909 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:22:51.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:51.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:51.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:51.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:52.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:52.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:52.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:52.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:52.348 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:52.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:52.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:52.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:52.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:52.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[107399]: pgmap v1223: 119 pgs: 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 3 remapped+peering, 5 unknown, 7 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 8586/126207 objects degraded (6.803%); 19504/126207 objects misplaced (15.454%); 30 MiB/s, 481 objects/s recovering 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[107399]: 12.15 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[107399]: 12.11 scrub starts 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[112165]: pgmap v1223: 119 pgs: 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 3 remapped+peering, 5 unknown, 7 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 8586/126207 objects degraded (6.803%); 19504/126207 objects misplaced (15.454%); 30 MiB/s, 481 objects/s recovering 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[112165]: 12.15 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:52 smithi001 ceph-mon[112165]: 12.11 scrub starts 2024-09-14T04:22:52.722 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:52.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:52.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:52 smithi012 ceph-mon[101180]: pgmap v1223: 119 pgs: 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 21 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 3 remapped+peering, 5 unknown, 7 active+undersized+degraded, 55 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 8586/126207 objects degraded (6.803%); 19504/126207 objects misplaced (15.454%); 30 MiB/s, 481 objects/s recovering 2024-09-14T04:22:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:52 smithi012 ceph-mon[101180]: 12.15 continuing backfill to osd.3 from (299'9438,340'15603] MIN to 340'15603 2024-09-14T04:22:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:52 smithi012 ceph-mon[101180]: 12.11 scrub starts 2024-09-14T04:22:52.867 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:52.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:52.994 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:52.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:52.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:53.095 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:53.118 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:53.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:53.226 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:22:53.226 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:22:53.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:53.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:53.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:53.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:53.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:53.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:53.515 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [5] live_osds: [1, 3, 4, 6, 7, 0, 2] 2024-09-14T04:22:53.515 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:22:53.515 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:22:53.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-09-14T04:22:53.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:53.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:53.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:53.746 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:53 smithi012 ceph-mon[101180]: 12.19 scrub starts 2024-09-14T04:22:53.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:53.855 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:53 smithi001 ceph-mon[107399]: 12.19 scrub starts 2024-09-14T04:22:53.855 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:53 smithi001 ceph-mon[112165]: 12.19 scrub starts 2024-09-14T04:22:53.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:53.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:53.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:53.999 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:54.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:54.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:54.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:54.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:54.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:54.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:54.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:54.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:54.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:54.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:54.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:54.757 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:54 smithi012 ceph-mon[101180]: pgmap v1224: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfilling, 22 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 4 remapped+peering, 7 active+undersized+degraded, 57 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 7238/125643 objects degraded (5.761%); 20475/125643 objects misplaced (16.296%); 21 MiB/s, 330 objects/s recovering 2024-09-14T04:22:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:54 smithi012 ceph-mon[101180]: 12.17 scrub starts 2024-09-14T04:22:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7238/125643 objects degraded (5.761%), 10 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:54.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:54.888 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:54.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[107399]: pgmap v1224: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfilling, 22 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 4 remapped+peering, 7 active+undersized+degraded, 57 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 7238/125643 objects degraded (5.761%); 20475/125643 objects misplaced (16.296%); 21 MiB/s, 330 objects/s recovering 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[107399]: 12.17 scrub starts 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7238/125643 objects degraded (5.761%), 10 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[112165]: pgmap v1224: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfilling, 22 active+clean+remapped, 3 active+remapped+backfill_wait, 22 active+undersized, 4 remapped+peering, 7 active+undersized+degraded, 57 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 7238/125643 objects degraded (5.761%); 20475/125643 objects misplaced (16.296%); 21 MiB/s, 330 objects/s recovering 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[112165]: 12.17 scrub starts 2024-09-14T04:22:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7238/125643 objects degraded (5.761%), 10 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:54.990 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:55.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:55.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:55.148 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:55.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:22:55.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:55.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:55.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:55.425 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:55.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:55.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:55.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:55.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:55.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:55.776 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:55.823 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:55.919 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:22:55.919 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:22:55.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:55.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:55.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:56.084 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:56.112 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:56.184 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:56.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:56.236 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:56.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:56.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:56.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:56.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:56.555 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:56 smithi001 ceph-mon[107399]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 23 active+clean+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 5950/125382 objects degraded (4.745%); 24805/125382 objects misplaced (19.784%); 29 MiB/s, 467 objects/s recovering 2024-09-14T04:22:56.555 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:56 smithi001 ceph-mon[112165]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 23 active+clean+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 5950/125382 objects degraded (4.745%); 24805/125382 objects misplaced (19.784%); 29 MiB/s, 467 objects/s recovering 2024-09-14T04:22:56.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:56.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:56.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:56.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:56.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:56 smithi012 ceph-mon[101180]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 activating+undersized+degraded+remapped, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 23 active+clean+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 5950/125382 objects degraded (4.745%); 24805/125382 objects misplaced (19.784%); 29 MiB/s, 467 objects/s recovering 2024-09-14T04:22:56.853 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:56.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:56.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:22:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:22:56] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.51.0" 2024-09-14T04:22:56.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:57.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.012 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:57.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:57.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:57.136 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:57.211 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:22:57.237 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:57.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:57.262 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:22:57.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:57.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:57.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:57.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:57.553 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:22:57.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:57.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:57.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:57.768 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:57.783 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2954468403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.783 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:57 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.784 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2954468403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.784 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:57 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2954468403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:57 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:22:57.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:57.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:57.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:57.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:22:58.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:22:58.124 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:22:58.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:22:58.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:58.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:58.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:58.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:58.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:58.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:58.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:58.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:58.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[112165]: pgmap v1226: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 7131/123645 objects degraded (5.767%); 25916/123645 objects misplaced (20.960%); 25 MiB/s, 398 objects/s recovering 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[112165]: osdmap e345: 8 total, 7 up, 6 in 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[107399]: pgmap v1226: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 7131/123645 objects degraded (5.767%); 25916/123645 objects misplaced (20.960%); 25 MiB/s, 398 objects/s recovering 2024-09-14T04:22:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:58.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:58.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:58 smithi001 ceph-mon[107399]: osdmap e345: 8 total, 7 up, 6 in 2024-09-14T04:22:58.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:58.783 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:58 smithi012 ceph-mon[101180]: pgmap v1226: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 56 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 7131/123645 objects degraded (5.767%); 25916/123645 objects misplaced (20.960%); 25 MiB/s, 398 objects/s recovering 2024-09-14T04:22:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:58 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:22:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:58 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:22:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:58 smithi012 ceph-mon[101180]: osdmap e345: 8 total, 7 up, 6 in 2024-09-14T04:22:58.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:58.909 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:58.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:59.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:22:59.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:22:59.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:59.160 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:59.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:59.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:59.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:59.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:22:59.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:22:59.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:22:59.544 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:22:59.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:59.669 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:59.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:22:59.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:22:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:22:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:22:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7131/123645 objects degraded (5.767%), 11 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:59.896 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:22:59.928 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:22:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:22:59.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:22:59.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:22:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7131/123645 objects degraded (5.767%), 11 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:22:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:22:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7131/123645 objects degraded (5.767%), 11 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:00.030 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:00.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:00.160 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:00.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:00.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:00.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:00.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:00.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:00.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:00.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:00.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:00.676 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:00.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:00.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:00.800 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:00.827 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:00.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:00 smithi012 ceph-mon[101180]: pgmap v1228: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 6984/122064 objects degraded (5.722%); 25541/122064 objects misplaced (20.924%); 14 MiB/s, 223 objects/s recovering 2024-09-14T04:23:00.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:00 smithi001 ceph-mon[107399]: pgmap v1228: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 6984/122064 objects degraded (5.722%); 25541/122064 objects misplaced (20.924%); 14 MiB/s, 223 objects/s recovering 2024-09-14T04:23:00.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:00 smithi001 ceph-mon[112165]: pgmap v1228: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 6984/122064 objects degraded (5.722%); 25541/122064 objects misplaced (20.924%); 14 MiB/s, 223 objects/s recovering 2024-09-14T04:23:00.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:00.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:00.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:01.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:01.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:01.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:01.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:01.212 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:23:01.212 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:23:01.251 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:23:01.252 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:23:01.283 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:23:01.283 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:23:01.310 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:23:01.311 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:23:01.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:01.339 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:23:01.339 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:23:01.340 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:23:01.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:01.366 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:23:01.366 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:23:01.390 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:23:01.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:23:01.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:01.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:01.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:01.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:01.709 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/597519700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/597519700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.748 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:01.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/597519700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:01.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:01.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:01.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:01.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:02.006 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:02.108 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:02.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:02.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:23:02.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:23:02.244 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:23:02.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:02.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:02.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:02.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:02.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:02.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:02.659 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:02.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:02.768 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:02 smithi001 ceph-mon[107399]: pgmap v1229: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 6873/119970 objects degraded (5.729%); 25029/119970 objects misplaced (20.863%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:23:02.769 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:02 smithi001 ceph-mon[112165]: pgmap v1229: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 6873/119970 objects degraded (5.729%); 25029/119970 objects misplaced (20.863%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:23:02.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:02 smithi012 ceph-mon[101180]: pgmap v1229: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 6873/119970 objects degraded (5.729%); 25029/119970 objects misplaced (20.863%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:23:02.887 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:02.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:02.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:03.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:03.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:03.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:03.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:03.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:03.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:03.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:03.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:03.519 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:03.520 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:03.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:03.564 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:03.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:03.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:03.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:03.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:03.941 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:03.967 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:03.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:04.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:04.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:04.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:04.218 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:04.245 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:23:04.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:04.344 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:04.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:04.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:moved 0 / 357 (0%) 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:avg 71.4 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 14.8459 -> 14.2857 (expected baseline 7.55778) 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.2 with 49 -> 49 pgs (0.686274 -> 0.686274 * mean) 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 75 -> 75 pgs (1.05042 -> 1.05042 * mean) 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 0.05 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0209 2024-09-14T04:23:04.551 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.2088 2024-09-14T04:23:04.552 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 0.7996 -> 0.8496 2024-09-14T04:23:04.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:04.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:04.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:04.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:04.732 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:04.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:04 smithi012 ceph-mon[101180]: pgmap v1230: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 6726/116874 objects degraded (5.755%); 24590/116874 objects misplaced (21.040%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:23:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55681'}"} : dispatch 2024-09-14T04:23:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6726/116874 objects degraded (5.755%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:04.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:04.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[107399]: pgmap v1230: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 6726/116874 objects degraded (5.755%); 24590/116874 objects misplaced (21.040%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55681'}"} : dispatch 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6726/116874 objects degraded (5.755%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[112165]: pgmap v1230: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 6726/116874 objects degraded (5.755%); 24590/116874 objects misplaced (21.040%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55681'}"} : dispatch 2024-09-14T04:23:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6726/116874 objects degraded (5.755%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:04.983 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:04.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:05.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:05.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:05.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:05.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:05.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:05.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:05.449 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:23:05.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-09-14T04:23:05.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:05.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:05.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:05.641 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:05.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[107399]: from='client.35988 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:05.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55681'}"}]': finished 2024-09-14T04:23:05.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[107399]: osdmap e346: 8 total, 7 up, 6 in 2024-09-14T04:23:05.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:05.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[112165]: from='client.35988 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:05.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55681'}"}]': finished 2024-09-14T04:23:05.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[112165]: osdmap e346: 8 total, 7 up, 6 in 2024-09-14T04:23:05.686 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:05.742 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:05.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:05 smithi012 ceph-mon[101180]: from='client.35988 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55681'}"}]': finished 2024-09-14T04:23:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:05 smithi012 ceph-mon[101180]: osdmap e346: 8 total, 7 up, 6 in 2024-09-14T04:23:05.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:05.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:05.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:05.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:06.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:06.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:06.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:06.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:06.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:06.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:06.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:06.429 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:06.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:06.553 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:06.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:06.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:06.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:06.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: pgmap v1232: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 6577/113790 objects degraded (5.780%); 23688/113790 objects misplaced (20.817%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: osdmap e347: 8 total, 7 up, 6 in 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15945 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15970 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: 12.16 continuing backfill to osd.3 from (301'10913,345'15945] MIN to 345'15945 2024-09-14T04:23:06.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[112165]: 12.6 continuing backfill to osd.3 from (301'10913,345'15970] MIN to 345'15970 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: pgmap v1232: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 6577/113790 objects degraded (5.780%); 23688/113790 objects misplaced (20.817%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: osdmap e347: 8 total, 7 up, 6 in 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15945 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15970 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: 12.16 continuing backfill to osd.3 from (301'10913,345'15945] MIN to 345'15945 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-mon[107399]: 12.6 continuing backfill to osd.3 from (301'10913,345'15970] MIN to 345'15970 2024-09-14T04:23:06.839 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:06] "GET /metrics HTTP/1.1" 200 36651 "" "Prometheus/2.51.0" 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: pgmap v1232: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 6577/113790 objects degraded (5.780%); 23688/113790 objects misplaced (20.817%); 1.2 MiB/s, 18 objects/s recovering 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: osdmap e347: 8 total, 7 up, 6 in 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15945 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 345'15970 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: 12.16 continuing backfill to osd.3 from (301'10913,345'15945] MIN to 345'15945 2024-09-14T04:23:06.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:06 smithi012 ceph-mon[101180]: 12.6 continuing backfill to osd.3 from (301'10913,345'15970] MIN to 345'15970 2024-09-14T04:23:06.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:06.952 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:06.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:07.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:07.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:07.118 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:07.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:07.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:07.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:07.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:07.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:07.496 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:07.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:07.620 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:07.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:07.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:07.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:07.875 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:07.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:07.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:07.977 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:08.004 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:08.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:08.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:08.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:08.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:08.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:08.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:08.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:08.492 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:08.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:08.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:08.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:08.660 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:08 smithi001 ceph-mon[107399]: pgmap v1234: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 6319/109953 objects degraded (5.747%); 23177/109953 objects misplaced (21.079%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:08.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:08 smithi001 ceph-mon[112165]: pgmap v1234: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 6319/109953 objects degraded (5.747%); 23177/109953 objects misplaced (21.079%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:08.753 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:08.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:08.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:08 smithi012 ceph-mon[101180]: pgmap v1234: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 6319/109953 objects degraded (5.747%); 23177/109953 objects misplaced (21.079%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:08.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:08.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:08.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:09.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:09.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:09.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:09.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:09.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:09.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:09.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:09.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:09.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.571 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:09.672 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:09.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:09.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:09.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6319/109953 objects degraded (5.747%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:09.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:09.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:09.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:09.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6319/109953 objects degraded (5.747%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:09.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6319/109953 objects degraded (5.747%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:10.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:10.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:10.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:10.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:10.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:10.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:10.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:10.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:10.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:10.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:10.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[107399]: pgmap v1235: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 6181/108291 objects degraded (5.708%); 22735/108291 objects misplaced (20.994%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[112165]: pgmap v1235: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 6181/108291 objects degraded (5.708%); 22735/108291 objects misplaced (20.994%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:10 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:10.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:10 smithi012 ceph-mon[101180]: pgmap v1235: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 6181/108291 objects degraded (5.708%); 22735/108291 objects misplaced (20.994%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:23:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:10 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:10 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:10.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:10.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:10.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:11.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:11.026 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:11.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:11.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:11.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:11.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:11.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:11.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:11.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:11.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:11.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:11.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:11.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:11.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:11.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:11.810 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:11.843 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[112165]: pgmap v1236: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 6063/106065 objects degraded (5.716%); 22171/106065 objects misplaced (20.903%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:23:11.843 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3255964396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.843 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.843 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[107399]: pgmap v1236: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 6063/106065 objects degraded (5.716%); 22171/106065 objects misplaced (20.903%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:23:11.844 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3255964396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.844 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:11 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:11 smithi012 ceph-mon[101180]: pgmap v1236: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 6063/106065 objects degraded (5.716%); 22171/106065 objects misplaced (20.903%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:23:11.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3255964396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:11 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:23:11.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:11.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:11.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:11.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:12.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:12.062 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:12.064 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:23:12.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:12.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:12.290 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:12.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:12.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:12.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:12.450 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:12.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:12.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:12.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:12.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:12.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:12.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: osdmap e348: 8 total, 7 up, 6 in 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:12.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: osdmap e348: 8 total, 7 up, 6 in 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:23:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: osdmap e348: 8 total, 7 up, 6 in 2024-09-14T04:23:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:12.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:12.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:12.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:13.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:13.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:13.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.217 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111739 11.2 11:4a5bd84c:::benchmark_data_smithi001_228075_object111738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 458.90254301599998, 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.00444102, 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111739 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.218 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429941+0000", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429943+0000", 2024-09-14T04:23:13.219 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.609e-06 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429968+0000", 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5333e-05 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429981+0000", 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2408999999999999e-05 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.220 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.429989+0000", 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3669999999999997e-06 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430098+0000", 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010843199999999999 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:23:13.221 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430104+0000", 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6599999999999998e-06 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.435778+0000", 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056739889999999999 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.222 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.435786+0000", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.542e-06 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.431760+0000", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99597435499999998 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.434382+0000", 2024-09-14T04:23:13.223 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026223240000000001 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111747 11.2 11:41cf9340:::benchmark_data_smithi001_228075_object111746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 458.90247488900002, 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.005084791, 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.224 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111747 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.225 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430009+0000", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430010+0000", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6899999999999997e-07 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430032+0000", 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1472e-05 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.226 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430039+0000", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9449999999999993e-06 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.430045+0000", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1030000000000001e-06 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431434+0000", 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001389415 2024-09-14T04:23:13.227 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431439+0000", 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2039999999999996e-06 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436670+0000", 2024-09-14T04:23:13.228 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052304860000000003 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436679+0000", 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9269999999999994e-06 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.434799+0000", 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99812029800000002 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.229 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.435094+0000", 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00029537200000000002 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111753 11.2 11:4fdf47a5:::benchmark_data_smithi001_228075_object111752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 458.90008328499999, 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.017779915, 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.230 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111753 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.231 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432401+0000", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432403+0000", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4219999999999999e-06 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432424+0000", 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1897999999999999e-05 2024-09-14T04:23:13.232 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432480+0000", 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5624999999999997e-05 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.432488+0000", 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.038e-06 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.233 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436702+0000", 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042138000000000002 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.436707+0000", 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9250000000000003e-06 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.435120+0000", 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99841299800000005 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.436382+0000", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012620559999999999 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.437292+0000", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00091012899999999997 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447762+0000", 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010469882999999999 2024-09-14T04:23:13.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.449982+0000", 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022205049999999998 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.450085+0000", 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010273200000000001 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.450181+0000", 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5903999999999995e-05 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145635 12.8 12:180d0589:::benchmark_data_smithi001_255951_object145634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 204.90157459, 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5285736509999999, 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.237 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145635 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430911+0000", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.593e-06 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430936+0000", 2024-09-14T04:23:13.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4176999999999999e-05 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430949+0000", 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3064000000000001e-05 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430957+0000", 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0010000000000001e-06 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431014+0000", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7336999999999997e-05 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431148+0000", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013379700000000001 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431286+0000", 2024-09-14T04:23:13.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013852299999999999 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.439229+0000", 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079427400000000002 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457555+0000", 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018325807 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.242 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457568+0000", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2772000000000001e-05 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.447670+0000", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99010279800000001 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.447682+0000", 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2094e-05 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959396+0000", 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.51171327 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959483+0000", 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7677999999999996e-05 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.244 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145637 12.8 12:1ca95b6c:::benchmark_data_smithi001_255951_object145636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 204.89980058200001, 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5270983039999999, 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.245 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145637 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432685+0000", 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5689999999999999e-06 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432710+0000", 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5086e-05 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432728+0000", 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7439999999999999e-05 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432735+0000", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9890000000000003e-06 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435266+0000", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025307289999999998 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435427+0000", 2024-09-14T04:23:13.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000161547 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435604+0000", 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000176935 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.442032+0000", 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064282209999999996 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.249 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457611+0000", 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015578843 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457622+0000", 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0392e-05 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448245+0000", 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99062326499999998 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448256+0000", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.112e-05 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959734+0000", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5114783090000001 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959782+0000", 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7858999999999997e-05 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.251 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145649 12.8 12:13d74b24:::benchmark_data_smithi001_255951_object145648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 204.871349399, 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.505730679, 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.252 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145649 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:23:13.253 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461136+0000", 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5090000000000001e-06 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461161+0000", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4749999999999999e-05 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461177+0000", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6164e-05 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461187+0000", 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5200000000000003e-06 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461244+0000", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6696000000000001e-05 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461253+0000", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0140000000000004e-06 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448292+0000", 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98703899100000003 2024-09-14T04:23:13.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448300+0000", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9910000000000002e-06 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959840+0000", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.511540222 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959989+0000", 2024-09-14T04:23:13.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014936699999999999 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.960181+0000", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019227099999999999 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966617+0000", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064353800000000001 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966635+0000", 2024-09-14T04:23:13.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.844e-05 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966682+0000", 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6220000000000001e-05 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966753+0000", 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1348999999999997e-05 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966866+0000", 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011279499999999999 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145657 12.8 12:110bae6a:::benchmark_data_smithi001_255951_object145656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:23:13.260 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 204.85882863500001, 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4938081240000001, 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145657 2024-09-14T04:23:13.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473658+0000", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8220000000000001e-06 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473682+0000", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4878000000000001e-05 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473698+0000", 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5522000000000001e-05 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473707+0000", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1530000000000003e-06 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473768+0000", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0510000000000002e-05 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473779+0000", 2024-09-14T04:23:13.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1559999999999999e-05 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448389+0000", 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97460960100000005 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448397+0000", 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3790000000000006e-06 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963298+0000", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5149011990000001 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963504+0000", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020578299999999999 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963697+0000", 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000192475 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966883+0000", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031863590000000002 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966911+0000", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7776999999999998e-05 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967313+0000", 2024-09-14T04:23:13.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040244800000000002 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967370+0000", 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6876000000000002e-05 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967464+0000", 2024-09-14T04:23:13.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3782000000000004e-05 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145630 12.b 12:d5814912:::benchmark_data_smithi001_255951_object145629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 203.89309383899999, 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5046016870000001, 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.269 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145630 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439392+0000", 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.508e-06 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439421+0000", 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8538e-05 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439438+0000", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7567999999999998e-05 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439450+0000", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1590000000000001e-05 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439594+0000", 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000144671 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439604+0000", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7070000000000004e-06 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444053+0000", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004448708 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444061+0000", 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3489999999999992e-06 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.941337+0000", 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4972761270000001 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.943992+0000", 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026549210000000002 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145648 12.b 12:de9db7e2:::benchmark_data_smithi001_255951_object145647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:23:13.274 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 203.89301344399999, 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5172668740000002, 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145648 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.275 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439472+0000", 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7800000000000001e-07 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439523+0000", 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1477000000000002e-05 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439533+0000", 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0023999999999999e-05 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.277 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439540+0000", 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8000000000000001e-06 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.278 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439720+0000", 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000180237 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439726+0000", 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1160000000000004e-06 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444090+0000", 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043639409999999997 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444097+0000", 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5189999999999998e-06 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944233+0000", 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.500135888 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.945064+0000", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00083095700000000001 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.945586+0000", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00052237900000000001 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.955563+0000", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099768319999999997 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956014+0000", 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00045092500000000002 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956568+0000", 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055460500000000001 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956625+0000", 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6283000000000003e-05 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.282 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956738+0000", 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113213 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:147029 12.1 12:84f6be8d:::benchmark_data_smithi001_255951_object1369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 200.25734836199999, 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22532575899999999, 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.283 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 147029 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075138+0000", 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7039999999999999e-06 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075143+0000", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3260000000000003e-06 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075159+0000", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5775999999999999e-05 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075166+0000", 2024-09-14T04:23:13.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3459999999999996e-06 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075219+0000", 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3031000000000001e-05 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075329+0000", 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011004400000000001 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:23:13.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075443+0000", 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113313 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.078989+0000", 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035467020000000001 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.079083+0000", 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3555999999999997e-05 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300371+0000", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22128830699999999 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300412+0000", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0707000000000002e-05 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300462+0000", 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9947000000000001e-05 2024-09-14T04:23:13.289 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:167946 12.b 12:da82cc1e:::benchmark_data_smithi001_255951_object22286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 176.19574348500001, 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21001428599999999, 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.290 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 167946 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.291 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136743+0000", 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.697e-06 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.292 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136749+0000", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9680000000000003e-06 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136764+0000", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5807e-05 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136772+0000", 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0229999999999992e-06 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136841+0000", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8594000000000001e-05 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136996+0000", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015524000000000001 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.137130+0000", 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013398199999999999 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.142972+0000", 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005841499 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.143744+0000", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00077236600000000002 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346552+0000", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20280820299999999 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346627+0000", 2024-09-14T04:23:13.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4492000000000004e-05 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346755+0000", 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000128415 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:182473 12.1 12:8e0dbc1e:::benchmark_data_smithi001_255951_object36813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 159.04163806899999, 2024-09-14T04:23:13.297 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.237500503, 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182473 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.298 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290848+0000", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.702e-06 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290854+0000", 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8989999999999998e-06 2024-09-14T04:23:13.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290870+0000", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6319000000000001e-05 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290880+0000", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5380000000000008e-06 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290948+0000", 2024-09-14T04:23:13.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8319999999999996e-05 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.291075+0000", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012682599999999999 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.291214+0000", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013861200000000001 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.294980+0000", 2024-09-14T04:23:13.301 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037662669999999998 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.295104+0000", 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123707 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528130+0000", 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23302693799999999 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.302 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528297+0000", 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016663499999999999 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528347+0000", 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9740000000000001e-05 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:182478 12.1 12:86b6932c:::benchmark_data_smithi001_255951_object36818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 159.03289146099999, 2024-09-14T04:23:13.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22889717000000001, 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182478 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.304 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.305 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.305 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:23:13.305 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.305 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299594+0000", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.55e-06 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299599+0000", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9259999999999999e-06 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299613+0000", 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3648e-05 2024-09-14T04:23:13.309 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299620+0000", 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0820000000000001e-06 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299674+0000", 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3477999999999999e-05 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299751+0000", 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7391999999999994e-05 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299818+0000", 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7144999999999995e-05 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.302548+0000", 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002730105 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.302679+0000", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013062999999999999 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528395+0000", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22571648499999999 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528451+0000", 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5522000000000001e-05 2024-09-14T04:23:13.312 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528490+0000", 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9206999999999999e-05 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246146 12.1 12:86d285cb:::benchmark_data_smithi001_255951_object100486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 40.528389015000002, 2024-09-14T04:23:13.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.70537081599999996, 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246146 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804097+0000", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2139999999999999e-06 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804102+0000", 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1560000000000003e-06 2024-09-14T04:23:13.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804115+0000", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3072e-05 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804122+0000", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.849e-06 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804175+0000", 2024-09-14T04:23:13.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3134000000000003e-05 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804180+0000", 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5799999999999999e-06 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500297+0000", 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.69611708299999997 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500422+0000", 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000124814 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500539+0000", 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011681299999999999 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503578+0000", 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00303915 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503636+0000", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7541999999999999e-05 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509290+0000", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056538700000000001 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509357+0000", 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6892999999999998e-05 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509466+0000", 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000109646 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246149 12.1 12:815a4314:::benchmark_data_smithi001_255951_object100489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 40.452302408000001, 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62963798900000001, 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.320 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246149 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.321 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880184+0000", 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8500000000000001e-06 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880191+0000", 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4139999999999997e-06 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880206+0000", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5213e-05 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880214+0000", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8699999999999992e-06 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880277+0000", 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2564999999999995e-05 2024-09-14T04:23:13.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880289+0000", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1912e-05 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503667+0000", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62337792000000003 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503790+0000", 2024-09-14T04:23:13.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012371800000000001 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503926+0000", 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135828 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509269+0000", 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005342935 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509528+0000", 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025847700000000001 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509684+0000", 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015632800000000001 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509732+0000", 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7926000000000002e-05 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509820+0000", 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8033000000000002e-05 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246168 12.1 12:80e4f094:::benchmark_data_smithi001_255951_object100508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 40.303670068999999, 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.48125710599999999, 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.327 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246168 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028815+0000", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.15e-06 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028820+0000", 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.51e-06 2024-09-14T04:23:13.329 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028839+0000", 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9276999999999999e-05 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028844+0000", 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8930000000000004e-06 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028906+0000", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1419000000000003e-05 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028917+0000", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1065e-05 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506507+0000", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.47759034299999997 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506620+0000", 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011265299999999999 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506740+0000", 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120423 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509580+0000", 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002840371 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509849+0000", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00026879699999999998 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509915+0000", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6259000000000004e-05 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509976+0000", 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0838999999999998e-05 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.333 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510071+0000", 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5106999999999996e-05 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246175 12.1 12:8f83bc3f:::benchmark_data_smithi001_255951_object100515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 40.242241341000003, 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.42337427, 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.334 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246175 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:23:13.335 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090245+0000", 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.562e-06 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090251+0000", 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1940000000000003e-06 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.336 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090265+0000", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4567e-05 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090274+0000", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6070000000000008e-06 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090339+0000", 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5283999999999994e-05 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.337 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090346+0000", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6109999999999999e-06 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510119+0000", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41977286000000003 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510221+0000", 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102538 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.338 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510288+0000", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6341999999999996e-05 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513037+0000", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027498499999999999 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513154+0000", 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116457 2024-09-14T04:23:13.339 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513461+0000", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030725199999999999 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513523+0000", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1754999999999994e-05 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513617+0000", 2024-09-14T04:23:13.340 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4390999999999998e-05 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:247454 12.1 12:83bf6cd5:::benchmark_data_smithi001_255951_object101794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:37.267794+0000", 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 34.064690859999999, 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20858391600000001, 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.341 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 247454 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267794+0000", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267794+0000", 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.342 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267795+0000", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3519999999999999e-06 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267800+0000", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1680000000000003e-06 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267811+0000", 2024-09-14T04:23:13.343 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1415e-05 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267820+0000", 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0539999999999995e-06 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267886+0000", 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6834000000000004e-05 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.344 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.267974+0000", 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7276999999999998e-05 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.268055+0000", 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1126999999999998e-05 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.269975+0000", 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001919965 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.345 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.270342+0000", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00036724000000000001 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.476234+0000", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20589211700000001 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.476278+0000", 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4375000000000001e-05 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:37.476377+0000", 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8992000000000004e-05 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:256578 12.8 12:16d08484:::benchmark_data_smithi001_255951_object110918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:09.569057+0000", 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 1.7634269840000001, 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.065486541999999995, 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.347 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 256578 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569057+0000", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.348 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569057+0000", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569059+0000", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.779e-06 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569065+0000", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2879999999999998e-06 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.349 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569081+0000", 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5818999999999999e-05 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569090+0000", 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6600000000000001e-06 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569157+0000", 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6931e-05 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569270+0000", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011287 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.569577+0000", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030691200000000001 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.572757+0000", 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031801249999999998 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.572794+0000", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7270000000000001e-05 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.634213+0000", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.06141862 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.634290+0000", 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7230000000000004e-05 2024-09-14T04:23:13.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.634544+0000", 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000254038 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:256591 12.8 12:1326d7d1:::benchmark_data_smithi001_255951_object110931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:09.596474+0000", 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 1.7360100869999999, 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.039540809000000003, 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:23:13.353 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 256591 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596474+0000", 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596474+0000", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596476+0000", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8470000000000001e-06 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596482+0000", 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9699999999999996e-06 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596496+0000", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3801000000000001e-05 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596504+0000", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6879999999999999e-06 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596561+0000", 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7544999999999999e-05 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596652+0000", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0556000000000005e-05 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.596890+0000", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00023862500000000001 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.598114+0000", 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001223568 2024-09-14T04:23:13.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.598766+0000", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00065256299999999999 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.635917+0000", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.037150216 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.635976+0000", 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9525999999999999e-05 2024-09-14T04:23:13.358 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:09.636015+0000", 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8903999999999999e-05 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:23:13.359 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:23:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:13.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:13.435 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:13.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:13.483 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:13.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:13.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:13.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:13.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:13.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:13.837 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:13.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:13.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:13.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:13 smithi001 ceph-mon[107399]: pgmap v1238: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 458 op/s; 5893/102786 objects degraded (5.733%); 21690/102786 objects misplaced (21.102%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:23:13.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:13 smithi001 ceph-mon[112165]: pgmap v1238: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 458 op/s; 5893/102786 objects degraded (5.733%); 21690/102786 objects misplaced (21.102%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:23:13.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:23:13.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:13.986 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:13 smithi012 ceph-mon[101180]: pgmap v1238: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 458 op/s; 5893/102786 objects degraded (5.733%); 21690/102786 objects misplaced (21.102%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:23:14.008 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:14.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:14.139 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:14.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:14.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:14.264 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:14.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:14.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:14.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:14.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:14.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:14.623 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:14.660 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:14.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:14.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:14.830 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5893/102786 objects degraded (5.733%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:14.831 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5893/102786 objects degraded (5.733%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:14.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:14.904 INFO:teuthology.orchestra.run.smithi001.stdout:moved 10 / 357 (2.80112%) 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:avg 71.4 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 14.2857 -> 13.4298 (expected baseline 7.55778) 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.2 with 49 -> 45 pgs (0.686274 -> 0.630252 * mean) 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 75 -> 73 pgs (1.05042 -> 1.02241 * mean) 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0194 2024-09-14T04:23:14.905 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.1942 2024-09-14T04:23:14.906 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 0.8496 -> 0.9931 2024-09-14T04:23:14.906 INFO:teuthology.orchestra.run.smithi001.stderr:no change 2024-09-14T04:23:14.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:14.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:15.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:15.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:15.074 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5893/102786 objects degraded (5.733%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:15.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:23:15.124 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:23:14.902+0000 7fc9b3f23640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-14T04:23:15.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:15.183 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:15.285 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:15.317 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:15.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:15.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:15.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:15.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:15.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:15.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:15.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:15 smithi001 ceph-mon[112165]: from='client.45944 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:15.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:15 smithi001 ceph-mon[112165]: pgmap v1239: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 5725/99423 objects degraded (5.758%); 20728/99423 objects misplaced (20.848%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:15 smithi001 ceph-mon[107399]: from='client.45944 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:15.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:15 smithi001 ceph-mon[107399]: pgmap v1239: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 5725/99423 objects degraded (5.758%); 20728/99423 objects misplaced (20.848%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:15.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:15.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:15 smithi012 ceph-mon[101180]: from='client.45944 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:15 smithi012 ceph-mon[101180]: pgmap v1239: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 5725/99423 objects degraded (5.758%); 20728/99423 objects misplaced (20.848%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:15.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:15.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:15.966 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:15.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:16.068 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:16.100 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:16.129 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:23:16.129 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:23:16.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:259882 12.b e348/312 12:d31c6fab:::benchmark_data_smithi001_255951_object114222:head v 348'16238, mlcod=348'16236)", 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:23:16.111457+0000", 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00070051300000000005, 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073024500000000003, 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111457+0000", 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111457+0000", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111457+0000", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111462+0000", 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8239999999999996e-06 2024-09-14T04:23:16.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111474+0000", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.185e-05 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111479+0000", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1109999999999999e-06 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111517+0000", 2024-09-14T04:23:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7693e-05 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111533+0000", 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5506999999999999e-05 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:259883 12.13 e348/341 12:cab620ba:::benchmark_data_smithi001_255951_object114223:head v 348'15627, mlcod=348'15625)", 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:23:16.111586+0000", 2024-09-14T04:23:16.133 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00057140399999999997, 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066301299999999995, 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111586+0000", 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111586+0000", 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111586+0000", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111593+0000", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111603+0000", 2024-09-14T04:23:16.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0386e-05 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111609+0000", 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9120000000000003e-06 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111635+0000", 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6092e-05 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:16.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:16.111650+0000", 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4854e-05 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 2 2024-09-14T04:23:16.137 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:23:16.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:16.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:16.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:16.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:16.460 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:16.496 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:16.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:16.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:16.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:16.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:16.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:16.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:23:16.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:16.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:16] "GET /metrics HTTP/1.1" 200 36651 "" "Prometheus/2.51.0" 2024-09-14T04:23:16.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:17.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:17.091 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:17.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:17.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.139 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:17.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:17.241 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:17.265 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:17.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:17.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:17.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:17.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:17.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:17.660 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:17.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:17.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:17.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:17.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:17.919 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:17.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:17.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:18.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:18.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:18.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:18.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:18.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:18.314 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:18.416 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:18.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:18.548 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:18 smithi012 ceph-mon[101180]: pgmap v1240: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5448/95070 objects degraded (5.731%); 20114/95070 objects misplaced (21.157%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:18.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:18.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:18 smithi001 ceph-mon[107399]: pgmap v1240: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5448/95070 objects degraded (5.731%); 20114/95070 objects misplaced (21.157%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:18 smithi001 ceph-mon[112165]: pgmap v1240: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5448/95070 objects degraded (5.731%); 20114/95070 objects misplaced (21.157%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:18.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:18.835 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:18.849 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:23:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:23:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:23:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:23:18.849 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:23:18.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:18.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:18.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:19.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:19.095 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:19.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:19.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:19.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:19.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:19.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:19.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:19.589 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:19.620 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:19.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:19.722 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:19.754 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:19.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:19 smithi012 ceph-mon[101180]: 12.9 scrub starts 2024-09-14T04:23:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5448/95070 objects degraded (5.731%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:19 smithi012 ceph-mon[101180]: 12.9 scrub ok 2024-09-14T04:23:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:19 smithi012 ceph-mon[101180]: 12.4 scrub starts 2024-09-14T04:23:19.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:19.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:19.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[107399]: 12.9 scrub starts 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5448/95070 objects degraded (5.731%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[107399]: 12.9 scrub ok 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[107399]: 12.4 scrub starts 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[112165]: 12.9 scrub starts 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5448/95070 objects degraded (5.731%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:19.924 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[112165]: 12.9 scrub ok 2024-09-14T04:23:19.925 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:19 smithi001 ceph-mon[112165]: 12.4 scrub starts 2024-09-14T04:23:19.961 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:23:19.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 3.0 2024-09-14T04:23:19.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:19.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:23:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:20.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:20.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:20.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:20.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:20.244 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:20.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:20.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:20.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:20.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:20.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:20.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:20.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:20.684 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: pgmap v1241: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5277/93006 objects degraded (5.674%); 19570/93006 objects misplaced (21.042%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:20.684 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.8 scrub starts 2024-09-14T04:23:20.684 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.4 scrub ok 2024-09-14T04:23:20.685 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.8 scrub ok 2024-09-14T04:23:20.685 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:20.685 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.7 scrub starts 2024-09-14T04:23:20.685 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.7 scrub ok 2024-09-14T04:23:20.685 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:20 smithi012 ceph-mon[101180]: 12.14 scrub starts 2024-09-14T04:23:20.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:20.719 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:20.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:20.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:20.853 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: pgmap v1241: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5277/93006 objects degraded (5.674%); 19570/93006 objects misplaced (21.042%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:20.853 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.8 scrub starts 2024-09-14T04:23:20.853 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.4 scrub ok 2024-09-14T04:23:20.853 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.8 scrub ok 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.7 scrub starts 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.7 scrub ok 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[107399]: 12.14 scrub starts 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: pgmap v1241: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5277/93006 objects degraded (5.674%); 19570/93006 objects misplaced (21.042%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.8 scrub starts 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.4 scrub ok 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.8 scrub ok 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.7 scrub starts 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.7 scrub ok 2024-09-14T04:23:20.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:20 smithi001 ceph-mon[112165]: 12.14 scrub starts 2024-09-14T04:23:20.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:20.970 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:20.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:21.065 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:23:21.065 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:23:21.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:21.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:21.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:21.099 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:23:21.099 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:23:21.130 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:23:21.130 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:23:21.161 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:23:21.161 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:23:21.192 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:23:21.192 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:23:21.193 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:23:21.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:21.223 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:23:21.223 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:23:21.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:21.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:21.253 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:23:21.254 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:23:21.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:21.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:21.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:21.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:21.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:21.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: 12.3 scrub starts 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: 12.3 scrub ok 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: 12.1 scrub starts 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: 12.14 scrub ok 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: 12.1 scrub ok 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2073161725' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: 12.3 scrub starts 2024-09-14T04:23:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: 12.3 scrub ok 2024-09-14T04:23:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: 12.1 scrub starts 2024-09-14T04:23:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: 12.14 scrub ok 2024-09-14T04:23:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: 12.1 scrub ok 2024-09-14T04:23:21.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2073161725' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:21.723 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:21.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:21.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: 12.3 scrub starts 2024-09-14T04:23:21.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: 12.3 scrub ok 2024-09-14T04:23:21.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: 12.1 scrub starts 2024-09-14T04:23:21.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: 12.14 scrub ok 2024-09-14T04:23:21.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: 12.1 scrub ok 2024-09-14T04:23:21.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2073161725' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:21.853 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:21.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:21.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:21.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:22.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:22.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:22.128 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:23:22.128 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:23:22.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:22.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:22.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:22.263 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:22.365 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:22.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:22.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:22.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:22.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:22.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:22.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:23:22.713 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:23:22.724 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:22.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:22.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:22.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:22 smithi012 ceph-mon[101180]: 12.13 scrub starts 2024-09-14T04:23:22.775 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:22 smithi012 ceph-mon[101180]: 12.13 scrub ok 2024-09-14T04:23:22.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:22 smithi012 ceph-mon[101180]: 12.18 scrub starts 2024-09-14T04:23:22.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:22 smithi012 ceph-mon[101180]: pgmap v1242: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 5152/90348 objects degraded (5.702%); 18899/90348 objects misplaced (20.918%) 2024-09-14T04:23:22.776 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:22 smithi012 ceph-mon[101180]: 12.18 scrub ok 2024-09-14T04:23:22.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[107399]: 12.13 scrub starts 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[107399]: 12.13 scrub ok 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[107399]: 12.18 scrub starts 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[107399]: pgmap v1242: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 5152/90348 objects degraded (5.702%); 18899/90348 objects misplaced (20.918%) 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[107399]: 12.18 scrub ok 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[112165]: 12.13 scrub starts 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[112165]: 12.13 scrub ok 2024-09-14T04:23:22.839 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[112165]: 12.18 scrub starts 2024-09-14T04:23:22.840 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[112165]: pgmap v1242: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 5152/90348 objects degraded (5.702%); 18899/90348 objects misplaced (20.918%) 2024-09-14T04:23:22.840 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:22 smithi001 ceph-mon[112165]: 12.18 scrub ok 2024-09-14T04:23:22.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:22.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:22.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:22.998 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:23.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:23.065 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:23.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:23.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:23.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:23.274 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:23.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:23.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:23.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:23.533 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:23.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:23.658 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:23.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[107399]: 12.11 scrub starts 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[107399]: 12.11 scrub ok 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[107399]: 12.19 scrub starts 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[107399]: 12.19 scrub ok 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[112165]: 12.11 scrub starts 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[112165]: 12.11 scrub ok 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[112165]: 12.19 scrub starts 2024-09-14T04:23:23.763 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:23 smithi001 ceph-mon[112165]: 12.19 scrub ok 2024-09-14T04:23:23.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:23 smithi012 ceph-mon[101180]: 12.11 scrub starts 2024-09-14T04:23:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:23 smithi012 ceph-mon[101180]: 12.11 scrub ok 2024-09-14T04:23:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:23 smithi012 ceph-mon[101180]: 12.19 scrub starts 2024-09-14T04:23:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:23 smithi012 ceph-mon[101180]: 12.19 scrub ok 2024-09-14T04:23:23.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:23.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:23.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:23.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:24.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:24.050 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:24.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:24.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:24.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:24.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:24.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:24.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:24.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:24.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:24.562 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:24.594 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:24 smithi012 ceph-mon[101180]: pgmap v1243: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 4987/86856 objects degraded (5.742%); 18412/86856 objects misplaced (21.198%) 2024-09-14T04:23:24.594 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:24 smithi012 ceph-mon[101180]: 12.17 scrub starts 2024-09-14T04:23:24.594 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4987/86856 objects degraded (5.742%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:24.594 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:24 smithi012 ceph-mon[101180]: 12.17 scrub ok 2024-09-14T04:23:24.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:24.696 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:24.714 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:23:24.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:24.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[107399]: pgmap v1243: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 4987/86856 objects degraded (5.742%); 18412/86856 objects misplaced (21.198%) 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[107399]: 12.17 scrub starts 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4987/86856 objects degraded (5.742%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[107399]: 12.17 scrub ok 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[112165]: pgmap v1243: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 4987/86856 objects degraded (5.742%); 18412/86856 objects misplaced (21.198%) 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[112165]: 12.17 scrub starts 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4987/86856 objects degraded (5.742%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:24.907 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:24 smithi001 ceph-mon[112165]: 12.17 scrub ok 2024-09-14T04:23:24.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:24.959 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:24.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:24.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:25.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:25.087 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:25.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:25.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:25.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:25.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:25.339 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:23:25.340 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:23:25.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:25.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:25.483 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:25.540 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:23:25.585 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:25.612 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:25.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:25.737 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:25.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:25.875 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:25.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:25.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:26.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:26.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:26.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:26.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:23:26.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:26.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:26.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:26.363 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:26.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:26.491 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:26.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:26.544 INFO:teuthology.orchestra.run.smithi001.stdout:moved 35 / 357 (9.80392%) 2024-09-14T04:23:26.544 INFO:teuthology.orchestra.run.smithi001.stdout:avg 71.4 2024-09-14T04:23:26.544 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 14.2857 -> 16.536 (expected baseline 7.55778) 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.2 with 49 -> 48 pgs (0.686274 -> 0.672269 * mean) 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.3 with 75 -> 64 pgs (1.05042 -> 0.896359 * mean) 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0177 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0195 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 weight 1.0000 -> 0.8106 2024-09-14T04:23:26.545 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 weight 1.0000 -> 0.8827 2024-09-14T04:23:26.546 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 0.8496 -> 0.9667 2024-09-14T04:23:26.546 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 weight 0.7887 -> 0.8092 2024-09-14T04:23:26.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:26.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:26.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:26.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:26.817 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:26 smithi012 ceph-mon[101180]: pgmap v1244: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 4793/83151 objects degraded (5.764%); 17430/83151 objects misplaced (20.962%) 2024-09-14T04:23:26.818 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:26 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:23:26.818 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:26.818 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:26 smithi012 ceph-mon[101180]: osdmap e349: 8 total, 7 up, 6 in 2024-09-14T04:23:26.818 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"} : dispatch 2024-09-14T04:23:26.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:26.919 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:26.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:26.933 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[107399]: pgmap v1244: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 4793/83151 objects degraded (5.764%); 17430/83151 objects misplaced (20.962%) 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[107399]: osdmap e349: 8 total, 7 up, 6 in 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"} : dispatch 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:26] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[112165]: pgmap v1244: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 4793/83151 objects degraded (5.764%); 17430/83151 objects misplaced (20.962%) 2024-09-14T04:23:26.934 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:23:26.935 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3896676793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:26.935 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[112165]: osdmap e349: 8 total, 7 up, 6 in 2024-09-14T04:23:26.935 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"} : dispatch 2024-09-14T04:23:27.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:27.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:27.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:27.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:27.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:27.284 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:23:27.285 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-09-14T04:23:27.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:27.295 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:27.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:27.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:27.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:27.545 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:27.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:27.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:27.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:27.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:27.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:27 smithi012 ceph-mon[101180]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:27 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"}]': finished 2024-09-14T04:23:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:27 smithi012 ceph-mon[101180]: osdmap e350: 8 total, 7 up, 6 in 2024-09-14T04:23:27.895 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:27.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:27.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[107399]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"}]': finished 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[107399]: osdmap e350: 8 total, 7 up, 6 in 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[112165]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'63356','2':'53033','3':'57849','4':'53124'}"}]': finished 2024-09-14T04:23:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:27 smithi001 ceph-mon[112165]: osdmap e350: 8 total, 7 up, 6 in 2024-09-14T04:23:28.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:28.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:28.113 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:23:28.113 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:23:28.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:28.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:28.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:28.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:28.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:28.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:28.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:28.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:23:28.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: pgmap v1247: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 4493/78852 objects degraded (5.698%); 13483/78852 objects misplaced (17.099%) 2024-09-14T04:23:28.633 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.6 continuing backfill to osd.1 from (301'10913,349'16378] MIN to 349'16378 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.16 continuing backfill to osd.1 from (301'10913,349'16286] MIN to 349'16286 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: osdmap e351: 8 total, 7 up, 6 in 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16013 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16231 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15952 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[112165]: 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16213 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: pgmap v1247: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 4493/78852 objects degraded (5.698%); 13483/78852 objects misplaced (17.099%) 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.6 continuing backfill to osd.1 from (301'10913,349'16378] MIN to 349'16378 2024-09-14T04:23:28.634 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.16 continuing backfill to osd.1 from (301'10913,349'16286] MIN to 349'16286 2024-09-14T04:23:28.635 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: osdmap e351: 8 total, 7 up, 6 in 2024-09-14T04:23:28.635 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16013 2024-09-14T04:23:28.635 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16231 2024-09-14T04:23:28.635 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15952 2024-09-14T04:23:28.635 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:28 smithi001 ceph-mon[107399]: 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16213 2024-09-14T04:23:28.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:28.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:28.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:28.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: pgmap v1247: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 22 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 4493/78852 objects degraded (5.698%); 13483/78852 objects misplaced (17.099%) 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.6 continuing backfill to osd.1 from (301'10913,349'16378] MIN to 349'16378 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.16 continuing backfill to osd.1 from (301'10913,349'16286] MIN to 349'16286 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: osdmap e351: 8 total, 7 up, 6 in 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16013 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16231 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15952 2024-09-14T04:23:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:28 smithi012 ceph-mon[101180]: 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 349'16213 2024-09-14T04:23:28.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:23:28.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:28.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:28.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:28.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:29.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:29.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:29.142 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:29.161 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:29.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:29.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:29.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:29.420 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:29.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:29.545 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:29.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:29.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.13 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15783 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: osdmap e352: 8 total, 7 up, 6 in 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15794 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16399 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.14 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16415 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.0 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15999 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: 12.10 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15950 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4493/78852 objects degraded (5.698%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:29.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[112165]: osdmap e353: 8 total, 7 up, 6 in 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.13 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15783 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: osdmap e352: 8 total, 7 up, 6 in 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15794 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16399 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.14 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16415 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.0 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15999 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: 12.10 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15950 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4493/78852 objects degraded (5.698%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:29.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:29 smithi001 ceph-mon[107399]: osdmap e353: 8 total, 7 up, 6 in 2024-09-14T04:23:29.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:29.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.13 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15783 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: osdmap e352: 8 total, 7 up, 6 in 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 349'15794 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16399 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.14 starting backfill to osd.0 from (0'0,0'0] MAX to 349'16415 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.0 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15999 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: 12.10 starting backfill to osd.3 from (0'0,0'0] MAX to 349'15950 2024-09-14T04:23:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4493/78852 objects degraded (5.698%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:29.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:29 smithi012 ceph-mon[101180]: osdmap e353: 8 total, 7 up, 6 in 2024-09-14T04:23:29.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:29.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:29.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:30.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:30.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:30.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:30.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:30.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:30.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:30.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:30.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:30.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:30.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:30.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:30.674 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:30 smithi001 ceph-mon[107399]: pgmap v1251: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 18 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 21 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 4387/77559 objects degraded (5.656%); 17928/77559 objects misplaced (23.115%) 2024-09-14T04:23:30.674 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:30 smithi001 ceph-mon[107399]: osdmap e354: 8 total, 7 up, 6 in 2024-09-14T04:23:30.675 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:30 smithi001 ceph-mon[112165]: pgmap v1251: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 18 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 21 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 4387/77559 objects degraded (5.656%); 17928/77559 objects misplaced (23.115%) 2024-09-14T04:23:30.675 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:30 smithi001 ceph-mon[112165]: osdmap e354: 8 total, 7 up, 6 in 2024-09-14T04:23:30.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:30.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:30.826 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:30 smithi012 ceph-mon[101180]: pgmap v1251: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped, 18 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 21 active+undersized, 9 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 4387/77559 objects degraded (5.656%); 17928/77559 objects misplaced (23.115%) 2024-09-14T04:23:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:30 smithi012 ceph-mon[101180]: osdmap e354: 8 total, 7 up, 6 in 2024-09-14T04:23:30.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:30.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:30.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:31.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:31.100 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:31.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:31.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:31.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:31.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:31.476 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:31.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:31.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.650 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:31.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:31.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:31.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:31.911 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:31.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:32.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:32.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:32.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:32.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:32.212 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:23:32.212 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:23:32.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:32.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:32.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:32.434 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:32.450 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:32.486 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:23:32.487 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:23:32.520 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:23:32.520 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:23:32.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:32.553 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:23:32.553 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:23:32.561 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:32.589 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:23:32.590 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:23:32.620 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:23:32.621 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:23:32.621 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:23:32.652 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:23:32.653 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:23:32.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:32.679 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:23:32.679 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:23:32.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:32 smithi001 ceph-mon[107399]: pgmap v1253: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 20 active+undersized, 7 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 48 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 4272/75984 objects degraded (5.622%); 19142/75984 objects misplaced (25.192%); 32 KiB/s, 0 objects/s recovering 2024-09-14T04:23:32.682 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/814392568' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:32.682 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:32 smithi001 ceph-mon[112165]: pgmap v1253: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 20 active+undersized, 7 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 48 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 4272/75984 objects degraded (5.622%); 19142/75984 objects misplaced (25.192%); 32 KiB/s, 0 objects/s recovering 2024-09-14T04:23:32.683 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/814392568' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:32.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:32.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:32.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:32 smithi012 ceph-mon[101180]: pgmap v1253: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 20 active+undersized, 7 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 48 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 4272/75984 objects degraded (5.622%); 19142/75984 objects misplaced (25.192%); 32 KiB/s, 0 objects/s recovering 2024-09-14T04:23:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/814392568' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:32.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:23:32.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:32.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:32.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:33.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:33.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:33.201 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:33.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:33.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:33.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:33.367 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:33.469 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:33.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:33.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:33.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:33.652 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:23:33.652 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:23:33.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:33.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:33.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:33.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:33.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:33.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:33.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:33 smithi001 ceph-mon[112165]: pgmap v1254: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 20 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 4180/72255 objects degraded (5.785%); 24748/72255 objects misplaced (34.251%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:33.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:33 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:23:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:33 smithi001 ceph-mon[107399]: pgmap v1254: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 20 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 4180/72255 objects degraded (5.785%); 24748/72255 objects misplaced (34.251%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:33 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:23:34.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:34.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:33 smithi012 ceph-mon[101180]: pgmap v1254: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 20 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 4180/72255 objects degraded (5.785%); 24748/72255 objects misplaced (34.251%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:33 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:23:34.151 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:34.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:34.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:34.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:34.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:34.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:34.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:34.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:moved 3 / 357 (0.840336%) 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:avg 71.4 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 16.536 -> 15.9987 (expected baseline 7.55778) 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.4 with 44 -> 43 pgs (0.616246 -> 0.602241 * mean) 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.1 with 73 -> 75 pgs (1.02241 -> 1.05042 * mean) 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:23:34.627 INFO:teuthology.orchestra.run.smithi001.stdout:oload 1000 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0167 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.1667 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 weight 0.8092 -> 0.9334 2024-09-14T04:23:34.628 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1 weight 0.9667 -> 1.0000 2024-09-14T04:23:34.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4180/72255 objects degraded (5.785%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:34.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[107399]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:34.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"} : dispatch 2024-09-14T04:23:34.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4180/72255 objects degraded (5.785%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:34.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[112165]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:34.647 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:34 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"} : dispatch 2024-09-14T04:23:34.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:34.704 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:34.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:34.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:34.810 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4180/72255 objects degraded (5.785%), 11 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:34.810 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:34 smithi012 ceph-mon[101180]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:34.810 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:34 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"} : dispatch 2024-09-14T04:23:34.836 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:34.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:34.937 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:34.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:34.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:35.067 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:23:35.067 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:23:35.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:270285 12.17 e355/350 12:ec79f638:::benchmark_data_smithi001_255951_object124625:head v 355'16352, mlcod=355'16350)", 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:23:35.064008+0000", 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0019696359999999999, 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019711970000000001, 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064008+0000", 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064008+0000", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064008+0000", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064015+0000", 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3409999999999997e-06 2024-09-14T04:23:35.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064026+0000", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1096e-05 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064032+0000", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.609e-06 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064073+0000", 2024-09-14T04:23:35.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1542999999999998e-05 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064098+0000", 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4258999999999999e-05 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.065945+0000", 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018469719999999999 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:35.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.065980+0000", 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4978999999999998e-05 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:270287 12.e e355/324 12:73e3e8c7:::benchmark_data_smithi001_255951_object124627:head v 355'16871, mlcod=355'16869)", 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:23:35.064503+0000", 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001474995, 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001580164, 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:23:35.072 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064503+0000", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064503+0000", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064502+0000", 2024-09-14T04:23:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064511+0000", 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2689999999999999e-06 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064526+0000", 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5409000000000001e-05 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:35.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064533+0000", 2024-09-14T04:23:35.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7499999999999997e-06 2024-09-14T04:23:35.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064596+0000", 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3459000000000003e-05 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:23:35.064628+0000", 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2020999999999997e-05 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:35.076 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:35.077 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:23:35.077 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 2 2024-09-14T04:23:35.077 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:23:35.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:35.118 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:35.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:35.252 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:35.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:35.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:35.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:35.500 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:23:35.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 3.0 2024-09-14T04:23:35.516 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:35.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:35.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:23:35.656 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:35.733 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"}]': finished 2024-09-14T04:23:35.733 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:35 smithi012 ceph-mon[101180]: osdmap e355: 8 total, 7 up, 6 in 2024-09-14T04:23:35.733 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:35.734 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:35.734 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:35 smithi012 ceph-mon[101180]: pgmap v1256: 119 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 17 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3964/68271 objects degraded (5.806%); 23176/68271 objects misplaced (33.947%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:23:35.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:23:35.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:35.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:35.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:35.928 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:35.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:35.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"}]': finished 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[107399]: osdmap e355: 8 total, 7 up, 6 in 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[107399]: pgmap v1256: 119 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 17 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3964/68271 objects degraded (5.806%); 23176/68271 objects misplaced (33.947%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536','2':'61173'}"}]': finished 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[112165]: osdmap e355: 8 total, 7 up, 6 in 2024-09-14T04:23:35.946 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:35.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:35.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:35 smithi001 ceph-mon[112165]: pgmap v1256: 119 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 17 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 50 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3964/68271 objects degraded (5.806%); 23176/68271 objects misplaced (33.947%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:23:35.982 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:36.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:36.073 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:36.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:36.199 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:36.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:36.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:36.323 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:36.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:36.367 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:36.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:36.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:36.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:36.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:36.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:36.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:36.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:36.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:36.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:36.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:36.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:36.967 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:36 smithi012 ceph-mon[101180]: osdmap e356: 8 total, 7 up, 6 in 2024-09-14T04:23:36.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:36 smithi001 ceph-mon[112165]: osdmap e356: 8 total, 7 up, 6 in 2024-09-14T04:23:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:36 smithi001 ceph-mon[107399]: osdmap e356: 8 total, 7 up, 6 in 2024-09-14T04:23:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:36] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-09-14T04:23:36.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:37.009 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:37.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:37.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:37.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:37.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:37.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:37.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:37.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:37.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:37.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:37.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:37 smithi001 ceph-mon[112165]: osdmap e357: 8 total, 7 up, 6 in 2024-09-14T04:23:37.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:37 smithi001 ceph-mon[112165]: pgmap v1259: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 1.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 3601/61887 objects degraded (5.819%); 21539/61887 objects misplaced (34.804%); 3.5 MiB/s, 54 objects/s recovering 2024-09-14T04:23:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:37 smithi001 ceph-mon[107399]: osdmap e357: 8 total, 7 up, 6 in 2024-09-14T04:23:37.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:37 smithi001 ceph-mon[107399]: pgmap v1259: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 1.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 3601/61887 objects degraded (5.819%); 21539/61887 objects misplaced (34.804%); 3.5 MiB/s, 54 objects/s recovering 2024-09-14T04:23:37.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.836 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:37.848 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:23:37.848 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:23:37.848 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:23:37.848 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:23:37.848 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:23:37.857 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:37 smithi012 ceph-mon[101180]: osdmap e357: 8 total, 7 up, 6 in 2024-09-14T04:23:37.857 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:37 smithi012 ceph-mon[101180]: pgmap v1259: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized+degraded, 51 active+clean; 1.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 3601/61887 objects degraded (5.819%); 21539/61887 objects misplaced (34.804%); 3.5 MiB/s, 54 objects/s recovering 2024-09-14T04:23:37.938 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:37.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:37.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:38.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:38.098 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:38.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:38.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:38.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:38.372 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:38.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:38.507 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:38.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:38.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:23:38.609 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:38.643 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:38.683 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:23:38.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:38.771 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:38.783 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:38 smithi012 ceph-mon[101180]: osdmap e358: 8 total, 7 up, 6 in 2024-09-14T04:23:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/828461836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:38 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:38.873 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:38.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:38.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[112165]: osdmap e358: 8 total, 7 up, 6 in 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/828461836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[107399]: osdmap e358: 8 total, 7 up, 6 in 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/828461836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:38 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:23:39.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:39.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:39.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:39.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:39.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:39.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:39.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:39.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:39.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:39.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:39.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:39.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:39.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:39.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:39.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:39 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:23:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:39 smithi012 ceph-mon[101180]: osdmap e359: 8 total, 7 up, 6 in 2024-09-14T04:23:39.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3601/61887 objects degraded (5.819%), 11 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:39.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:39 smithi012 ceph-mon[101180]: pgmap v1262: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 51 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 3403/59697 objects degraded (5.700%); 20642/59697 objects misplaced (34.578%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:39.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:39.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:39.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[107399]: osdmap e359: 8 total, 7 up, 6 in 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3601/61887 objects degraded (5.819%), 11 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[107399]: pgmap v1262: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 51 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 3403/59697 objects degraded (5.700%); 20642/59697 objects misplaced (34.578%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:23:39.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[112165]: osdmap e359: 8 total, 7 up, 6 in 2024-09-14T04:23:39.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3601/61887 objects degraded (5.819%), 11 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:39.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:39 smithi001 ceph-mon[112165]: pgmap v1262: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 23 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 51 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 3403/59697 objects degraded (5.700%); 20642/59697 objects misplaced (34.578%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:23:40.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:40.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:40.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:40.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:40.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:40.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:40.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:40.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:40.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:40.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:40.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:40.578 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:40.609 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:40.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:40.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:40.799 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:23:40.799 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:23:40.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:23:40.799 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:23:40.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111749 11.f 11:fa92e749:::benchmark_data_smithi001_228075_object111748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 488.37363407300001, 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0160433040000001, 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111749 2024-09-14T04:23:40.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421073+0000", 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421072+0000", 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421102+0000", 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0614000000000002e-05 2024-09-14T04:23:40.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421115+0000", 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2445e-05 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421122+0000", 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7770000000000002e-06 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421176+0000", 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3489000000000002e-05 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421337+0000", 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016110899999999999 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.421484+0000", 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147243 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435948+0000", 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014464069 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.435958+0000", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7450000000000007e-06 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437034+0000", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.001076007 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437116+0000", 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2194999999999995e-05 2024-09-14T04:23:40.806 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111769 11.f 11:f6a7532d:::benchmark_data_smithi001_228075_object111768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 488.32617783900002, 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98091847099999996, 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.807 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111769 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:23:40.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468529+0000", 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468528+0000", 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468582+0000", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4372000000000001e-05 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468594+0000", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.19e-05 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468600+0000", 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9710000000000001e-06 2024-09-14T04:23:40.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468652+0000", 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1994999999999999e-05 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.468663+0000", 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437585+0000", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96892259300000005 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437788+0000", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000203069 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.437921+0000", 2024-09-14T04:23:40.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013228200000000001 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.442200+0000", 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042791950000000004 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449285+0000", 2024-09-14T04:23:40.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070849850000000002 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449347+0000", 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1766000000000005e-05 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.449448+0000", 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010103 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.814 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:111770 11.4 11:2369255e:::benchmark_data_smithi001_228075_object111769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 488.323977791, 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97033377499999995, 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111770 2024-09-14T04:23:40.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470729+0000", 2024-09-14T04:23:40.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470728+0000", 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470785+0000", 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6273999999999997e-05 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470795+0000", 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0738e-05 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470801+0000", 2024-09-14T04:23:40.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1869999999999996e-06 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470850+0000", 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9265999999999999e-05 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:32.470857+0000", 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8770000000000004e-06 2024-09-14T04:23:40.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433741+0000", 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96288411200000001 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433867+0000", 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125864 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6", 2024-09-14T04:23:40.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.433977+0000", 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010985 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.439362+0000", 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053849340000000001 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440897+0000", 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015350260000000001 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.440984+0000", 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7182000000000007e-05 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:33.441063+0000", 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9246999999999995e-05 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117402 11.3 11:c184ceb7:::benchmark_data_smithi001_228075_object117401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:23:40.822 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 481.245817649, 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.27739556599999998, 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117402 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548889+0000", 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548888+0000", 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548957+0000", 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8973999999999994e-05 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548974+0000", 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7041000000000002e-05 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.548982+0000", 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8639999999999996e-06 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.550732+0000", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017508090000000001 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.550945+0000", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000212924 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.551141+0000", 2024-09-14T04:23:40.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019523399999999999 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.555822+0000", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004681485 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.561382+0000", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005560031 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823453+0000", 2024-09-14T04:23:40.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26207078 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823510+0000", 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6733e-05 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.826285+0000", 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027753119999999998 2024-09-14T04:23:40.828 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117407 11.5 11:aca3a4ce:::benchmark_data_smithi001_228075_object117406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 481.23804266799999, 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26693572100000001, 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.829 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117407 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:23:40.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556664+0000", 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.556663+0000", 2024-09-14T04:23:40.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557270+0000", 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00060745000000000005 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557287+0000", 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6569999999999999e-05 2024-09-14T04:23:40.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557294+0000", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4340000000000003e-06 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557341+0000", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6440000000000003e-05 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557525+0000", 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018454499999999999 2024-09-14T04:23:40.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.557650+0000", 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012483500000000001 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.564719+0000", 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070693259999999999 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.571495+0000", 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067759969999999997 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823510+0000", 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.252014234 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823557+0000", 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7543999999999997e-05 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.823600+0000", 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2947000000000001e-05 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117429 11.3 11:c5090e60:::benchmark_data_smithi001_228075_object117428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:23:40.837 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 481.22010657800001, 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.294253918, 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117429 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574600+0000", 2024-09-14T04:23:40.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574599+0000", 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574640+0000", 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0890000000000003e-05 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574657+0000", 2024-09-14T04:23:40.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7133999999999999e-05 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574664+0000", 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5700000000000004e-06 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574733+0000", 2024-09-14T04:23:40.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8610999999999998e-05 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.574937+0000", 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020398599999999999 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.575163+0000", 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000226166 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.579049+0000", 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038853379999999999 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.580021+0000", 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00097227100000000003 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.865872+0000", 2024-09-14T04:23:40.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28585137599999999 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.865934+0000", 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2028999999999999e-05 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868854+0000", 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029201040000000002 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35451.0:117447 11.3 11:c9a66c9a:::benchmark_data_smithi001_228075_object117446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 481.193257765, 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26974246400000002, 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.846 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35451", 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117447 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601449+0000", 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601448+0000", 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601496+0000", 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8690000000000003e-05 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601509+0000", 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2318e-05 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601515+0000", 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.781e-06 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601562+0000", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6851000000000003e-05 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601707+0000", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014509899999999999 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.601815+0000", 2024-09-14T04:23:40.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107524 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.604811+0000", 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029959470000000001 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.605687+0000", 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087606800000000003 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868906+0000", 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26321921999999998 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.868958+0000", 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2215000000000001e-05 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:15:39.871192+0000", 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022335160000000001 2024-09-14T04:23:40.852 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145636 12.d 12:b54dcd2d:::benchmark_data_smithi001_255951_object145635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 234.36279528599999, 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6968347069999998, 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.853 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145636 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:23:40.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431910+0000", 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431939+0000", 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8583000000000001e-05 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431948+0000", 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2059999999999996e-06 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431954+0000", 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6269999999999997e-06 2024-09-14T04:23:40.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432030+0000", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5821000000000005e-05 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432175+0000", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014539199999999999 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432308+0000", 2024-09-14T04:23:40.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013317900000000001 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.441523+0000", 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092149239999999993 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.449041+0000", 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00751791 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.449062+0000", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0616999999999999e-05 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434784+0000", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98572267800000002 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434800+0000", 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5634e-05 2024-09-14T04:23:40.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128682+0000", 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6938822039999999 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128746+0000", 2024-09-14T04:23:40.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4221999999999996e-05 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145654 12.d 12:b3f58279:::benchmark_data_smithi001_255951_object145653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 234.32601020000001, 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.664128394, 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.861 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145654 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468695+0000", 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468737+0000", 2024-09-14T04:23:40.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1696000000000002e-05 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468747+0000", 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8840000000000006e-06 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468752+0000", 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2320000000000001e-06 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468800+0000", 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8006000000000001e-05 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468807+0000", 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7009999999999999e-06 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434836+0000", 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96602920299999995 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434846+0000", 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0328e-05 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128937+0000", 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6940900490000002 2024-09-14T04:23:40.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.129068+0000", 2024-09-14T04:23:40.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013180099999999999 2024-09-14T04:23:40.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.129192+0000", 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012386299999999999 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132186+0000", 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029939110000000001 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132344+0000", 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157539 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132719+0000", 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037569300000000002 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132757+0000", 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7171e-05 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132825+0000", 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8707999999999993e-05 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.871 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145656 12.7 12:e2152f9f:::benchmark_data_smithi001_255951_object145655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 234.32318938700001, 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4715065950000001, 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145656 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471516+0000", 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471570+0000", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3325999999999997e-05 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471581+0000", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1319e-05 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471588+0000", 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7089999999999996e-06 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471626+0000", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7920000000000003e-05 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471633+0000", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0330000000000003e-06 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.435398+0000", 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96376509200000005 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.435407+0000", 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4560000000000006e-06 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939054+0000", 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5036471769999999 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939205+0000", 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015092799999999999 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939321+0000", 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116131 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942283+0000", 2024-09-14T04:23:40.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029620380000000002 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942614+0000", 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000330861 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942930+0000", 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031613099999999999 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942967+0000", 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6519000000000001e-05 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.943024+0000", 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7374999999999999e-05 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169693 12.e e301/283 12:7fb05d18:::benchmark_data_smithi001_255951_object24033:head v 301'10537, mlcod=301'10533)", 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:23:40.879 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.338873904, 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38418591800000002, 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455836+0000", 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4510000000000001e-06 2024-09-14T04:23:40.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455843+0000", 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4989999999999999e-06 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455846+0000", 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.569e-06 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.465968+0000", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010121404000000001 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.465987+0000", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9069e-05 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838217+0000", 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37223062200000001 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840019+0000", 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00180157 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169704 12.2 e301/283 12:4868bfdc:::benchmark_data_smithi001_255951_object24044:head v 301'10684, mlcod=301'10682)", 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.33097588300001, 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.376235192, 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.884 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463730+0000", 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463738+0000", 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463749+0000", 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0523000000000001e-05 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463755+0000", 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2199999999999997e-06 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463795+0000", 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0185999999999998e-05 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463813+0000", 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8094999999999999e-05 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838212+0000", 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37439889199999998 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.839966+0000", 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017544469999999999 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169707 12.0 e301/298 12:0f3a37da:::benchmark_data_smithi001_255951_object24047:head v 301'10515, mlcod=301'10513)", 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.330045294, 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37545621800000001, 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464661+0000", 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:23:40.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464667+0000", 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0279999999999997e-06 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464686+0000", 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8640000000000001e-05 2024-09-14T04:23:40.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464692+0000", 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8150000000000002e-06 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464725+0000", 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3404000000000001e-05 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464746+0000", 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0526999999999998e-05 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838191+0000", 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.373445845 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840118+0000", 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001926599 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169710 12.2 e301/283 12:4ac1415e:::benchmark_data_smithi001_255951_object24050:head v 301'10685, mlcod=301'10682)", 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.32724103699999, 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37253622600000003, 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:23:40.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467465+0000", 2024-09-14T04:23:40.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467474+0000", 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2029999999999998e-06 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467486+0000", 2024-09-14T04:23:40.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1875999999999999e-05 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467493+0000", 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6610000000000003e-06 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467530+0000", 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7261000000000001e-05 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467548+0000", 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8037999999999999e-05 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.839973+0000", 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37242533 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840002+0000", 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8946999999999999e-05 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169711 12.0 e301/298 12:0996da4d:::benchmark_data_smithi001_255951_object24051:head v 301'10516, mlcod=301'10513)", 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.32670698699999, 2024-09-14T04:23:40.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37581841599999999, 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467999+0000", 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468006+0000", 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1339999999999998e-06 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468016+0000", 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.026e-05 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468023+0000", 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3439999999999997e-06 2024-09-14T04:23:40.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468100+0000", 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6761999999999995e-05 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468120+0000", 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0733e-05 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.843700+0000", 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37557949099999999 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.843818+0000", 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118651 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:169713 12.7 12:ec3b0951:::benchmark_data_smithi001_255951_object24053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.32615803499999, 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38481411700000001, 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 169713 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468547+0000", 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:23:40.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468554+0000", 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4729999999999997e-06 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468567+0000", 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3057000000000001e-05 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468574+0000", 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2420000000000003e-06 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.469065+0000", 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00049128599999999996 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.469183+0000", 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117336 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840086+0000", 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37090367000000002 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.851198+0000", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011112260000000001 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.851461+0000", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026300800000000001 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853194+0000", 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001732555 2024-09-14T04:23:40.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853268+0000", 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3950999999999997e-05 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853363+0000", 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5117000000000004e-05 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169716 12.1 e301/283 12:8a7b11ab:::benchmark_data_smithi001_255951_object24056:head v 301'10620, mlcod=301'10618)", 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.32360051699999, 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38175547799999998, 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:23:40.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471106+0000", 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471113+0000", 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.486e-06 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838195+0000", 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.367081775 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838211+0000", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5896000000000002e-05 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838254+0000", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2775000000000003e-05 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838278+0000", 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4409e-05 2024-09-14T04:23:40.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.852789+0000", 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014511082 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.852862+0000", 2024-09-14T04:23:40.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2694000000000006e-05 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169712 12.0 e301/298 12:07f327ad:::benchmark_data_smithi001_255951_object24052:head v 301'10517, mlcod=301'10513)", 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 203.32288133099999, 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37244635700000001, 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471825+0000", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471832+0000", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2640000000000002e-06 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838208+0000", 2024-09-14T04:23:40.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.36637539400000002 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838216+0000", 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.67e-06 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840153+0000", 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019370069999999999 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840173+0000", 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0157e-05 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.844245+0000", 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040714260000000004 2024-09-14T04:23:40.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.844272+0000", 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7202000000000001e-05 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:242450 12.d 12:b7679709:::benchmark_data_smithi001_255951_object96790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.200105851999993, 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28077718200000001, 2024-09-14T04:23:40.923 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 242450 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594600+0000", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594606+0000", 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5810000000000002e-06 2024-09-14T04:23:40.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594621+0000", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5125e-05 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594629+0000", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9370000000000004e-06 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594687+0000", 2024-09-14T04:23:40.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8149000000000002e-05 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594834+0000", 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014639899999999999 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594990+0000", 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015630400000000001 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.602164+0000", 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071733379999999996 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.602322+0000", 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000158171 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875247+0000", 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.27292567000000001 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875288+0000", 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0624999999999998e-05 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875378+0000", 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0378999999999996e-05 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:243548 12.7 12:ed83732d:::benchmark_data_smithi001_255951_object97888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-14T04:23:40.930 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 90.392973454, 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96499681999999998, 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 243548 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:23:40.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401731+0000", 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401744+0000", 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2456999999999999e-05 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401774+0000", 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0086e-05 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401787+0000", 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3644e-05 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401910+0000", 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012274000000000001 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401918+0000", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.361975+0000", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96005681700000001 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362097+0000", 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012235999999999999 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362195+0000", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8171000000000006e-05 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.365736+0000", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035408940000000002 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366255+0000", 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051849299999999997 2024-09-14T04:23:40.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366541+0000", 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028677599999999998 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366603+0000", 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2151999999999997e-05 2024-09-14T04:23:40.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366730+0000", 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012696499999999999 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:23:40.938 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:23:40.969 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:40.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:40.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:40 smithi001 ceph-mon[112165]: osdmap e360: 8 total, 7 up, 6 in 2024-09-14T04:23:40.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:40 smithi001 ceph-mon[107399]: osdmap e360: 8 total, 7 up, 6 in 2024-09-14T04:23:41.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:40 smithi012 ceph-mon[101180]: osdmap e360: 8 total, 7 up, 6 in 2024-09-14T04:23:41.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:41.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:41.223 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:41.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:41.347 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:41.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:41.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:41.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:41.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:41.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:41.724 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:41.735 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:23:41.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:41.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:41.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:41 smithi001 ceph-mon[112165]: osdmap e361: 8 total, 7 up, 6 in 2024-09-14T04:23:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:41 smithi001 ceph-mon[112165]: pgmap v1265: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 3279/57591 objects degraded (5.694%); 18839/57591 objects misplaced (32.712%); 19 MiB/s, 307 objects/s recovering 2024-09-14T04:23:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:41 smithi001 ceph-mon[107399]: osdmap e361: 8 total, 7 up, 6 in 2024-09-14T04:23:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:41 smithi001 ceph-mon[107399]: pgmap v1265: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 3279/57591 objects degraded (5.694%); 18839/57591 objects misplaced (32.712%); 19 MiB/s, 307 objects/s recovering 2024-09-14T04:23:41.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:42.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:42.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:41 smithi012 ceph-mon[101180]: osdmap e361: 8 total, 7 up, 6 in 2024-09-14T04:23:42.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:41 smithi012 ceph-mon[101180]: pgmap v1265: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 3279/57591 objects degraded (5.694%); 18839/57591 objects misplaced (32.712%); 19 MiB/s, 307 objects/s recovering 2024-09-14T04:23:42.123 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:42.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:42.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:42.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:42.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:42.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:42.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:42.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:42.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:42.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:42.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:42.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:42.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:42.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:42.914 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:42.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:42.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:42 smithi001 ceph-mon[107399]: osdmap e362: 8 total, 7 up, 6 in 2024-09-14T04:23:42.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:42 smithi001 ceph-mon[112165]: osdmap e362: 8 total, 7 up, 6 in 2024-09-14T04:23:43.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:43.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:42 smithi012 ceph-mon[101180]: osdmap e362: 8 total, 7 up, 6 in 2024-09-14T04:23:43.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:43.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:43.280 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:43.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:43.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:43.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:43.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:43.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:43.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:43.695 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:43.797 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:43.838 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:43.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:43.969 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:43.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:43.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:43 smithi001 ceph-mon[107399]: pgmap v1267: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 3082/53112 objects degraded (5.803%); 17755/53112 objects misplaced (33.429%); 19 MiB/s, 300 objects/s recovering 2024-09-14T04:23:43.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:43 smithi001 ceph-mon[112165]: pgmap v1267: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 3082/53112 objects degraded (5.803%); 17755/53112 objects misplaced (33.429%); 19 MiB/s, 300 objects/s recovering 2024-09-14T04:23:44.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:43 smithi012 ceph-mon[101180]: pgmap v1267: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized+degraded, 54 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 3082/53112 objects degraded (5.803%); 17755/53112 objects misplaced (33.429%); 19 MiB/s, 300 objects/s recovering 2024-09-14T04:23:44.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:44.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:44.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:44.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:44.361 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:44.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:44.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:44.498 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:44.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:44.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:44.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:44.766 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:44.868 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:44.898 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:44.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:44.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:44.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3082/53112 objects degraded (5.803%), 10 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:44.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3082/53112 objects degraded (5.803%), 10 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:45.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:45.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3082/53112 objects degraded (5.803%), 10 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:45.125 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:45.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:45.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:45.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:45.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:45.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:45.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:45.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:45.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:45.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:45.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:45.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:45.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:45.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:46.015 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:45 smithi001 ceph-mon[107399]: pgmap v1268: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 6 active+undersized+degraded, 54 active+clean; 1.0 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 2887/49887 objects degraded (5.787%); 16526/49887 objects misplaced (33.127%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:23:46.016 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:45 smithi001 ceph-mon[112165]: pgmap v1268: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 6 active+undersized+degraded, 54 active+clean; 1.0 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 2887/49887 objects degraded (5.787%); 16526/49887 objects misplaced (33.127%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:23:46.089 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:45 smithi012 ceph-mon[101180]: pgmap v1268: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 6 active+undersized+degraded, 54 active+clean; 1.0 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 2887/49887 objects degraded (5.787%); 16526/49887 objects misplaced (33.127%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:23:46.142 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:46.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:46.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:46.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:46.366 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:23:46.366 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:23:46.366 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:23:46.367 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:23:46.368 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:23:46.368 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:23:46.368 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:23:46.368 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:23:46.368 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:23:46.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:46.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:46.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:46.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:46.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:46.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:46.726 INFO:teuthology.orchestra.run.smithi001.stdout:moved 10 / 357 (2.80112%) 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:avg 71.4 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:stddev 15.9987 -> 15.0532 (expected baseline 7.55778) 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:min osd.4 with 43 -> 42 pgs (0.602241 -> 0.588235 * mean) 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:max osd.1 with 75 -> 76 pgs (1.05042 -> 1.06443 * mean) 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:oload 110 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:max_change 3 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:max_change_osds 4 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:average_utilization 0.0146 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:overload_utilization 0.0161 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5 weight 1.0000 -> 0.7620 2024-09-14T04:23:46.727 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2 weight 0.9334 -> 1.0000 2024-09-14T04:23:46.728 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4 weight 0.8106 -> 0.6745 2024-09-14T04:23:46.728 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3 weight 0.8827 -> 0.8016 2024-09-14T04:23:46.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:46.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:46.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:46.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:46.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1491826665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"} : dispatch 2024-09-14T04:23:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:46] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-09-14T04:23:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1491826665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"} : dispatch 2024-09-14T04:23:47.018 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:47.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1491826665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:23:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"} : dispatch 2024-09-14T04:23:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:23:47.149 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:23:47.154 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:47.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:47.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:47.319 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:47.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:47.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:47.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:47.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:47.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:47.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:47.795 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:47.818 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:47.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[112165]: from='client.36033 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:47.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"}]': finished 2024-09-14T04:23:47.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[112165]: osdmap e363: 8 total, 7 up, 6 in 2024-09-14T04:23:47.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[112165]: pgmap v1270: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 5 active+undersized+degraded, 53 active+clean; 921 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 2591/44274 objects degraded (5.852%); 15252/44274 objects misplaced (34.449%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:23:47.884 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[112165]: osdmap e364: 8 total, 7 up, 6 in 2024-09-14T04:23:47.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[107399]: from='client.36033 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:47.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"}]': finished 2024-09-14T04:23:47.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[107399]: osdmap e363: 8 total, 7 up, 6 in 2024-09-14T04:23:47.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[107399]: pgmap v1270: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 5 active+undersized+degraded, 53 active+clean; 921 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 2591/44274 objects degraded (5.852%); 15252/44274 objects misplaced (34.449%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:23:47.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:47 smithi001 ceph-mon[107399]: osdmap e364: 8 total, 7 up, 6 in 2024-09-14T04:23:47.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:47.925 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:47.944 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:47.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:48.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:48.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:23:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:47 smithi012 ceph-mon[101180]: from='client.36033 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:23:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','3':'52534','4':'44207','5':'49937'}"}]': finished 2024-09-14T04:23:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:47 smithi012 ceph-mon[101180]: osdmap e363: 8 total, 7 up, 6 in 2024-09-14T04:23:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:47 smithi012 ceph-mon[101180]: pgmap v1270: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 25 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 5 active+undersized+degraded, 53 active+clean; 921 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 2591/44274 objects degraded (5.852%); 15252/44274 objects misplaced (34.449%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:23:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:47 smithi012 ceph-mon[101180]: osdmap e364: 8 total, 7 up, 6 in 2024-09-14T04:23:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.100 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:48.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:48.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:48.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:48.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:48.368 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:48.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:48.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:48.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:48.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:48.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:48.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:48.762 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:48.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:48.891 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:48.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:48.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:49.020 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:48 smithi012 ceph-mon[101180]: 12.f starting backfill to osd.2 from (0'0,0'0] MAX to 362'17158 2024-09-14T04:23:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:48 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:48 smithi012 ceph-mon[101180]: osdmap e365: 8 total, 7 up, 6 in 2024-09-14T04:23:49.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[107399]: 12.f starting backfill to osd.2 from (0'0,0'0] MAX to 362'17158 2024-09-14T04:23:49.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:49.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[107399]: osdmap e365: 8 total, 7 up, 6 in 2024-09-14T04:23:49.100 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[112165]: 12.f starting backfill to osd.2 from (0'0,0'0] MAX to 362'17158 2024-09-14T04:23:49.100 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:23:49.100 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:48 smithi001 ceph-mon[112165]: osdmap e365: 8 total, 7 up, 6 in 2024-09-14T04:23:49.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:49.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:49.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:23:49.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:49.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:49.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:49.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:49.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:49.522 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:49.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:49.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:49.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:49.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:49.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:49.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:49.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:50.010 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:50.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2591/44274 objects degraded (5.852%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:49 smithi012 ceph-mon[101180]: osdmap e366: 8 total, 7 up, 6 in 2024-09-14T04:23:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:49 smithi012 ceph-mon[101180]: pgmap v1274: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 53 active+clean; 874 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 2397/42027 objects degraded (5.703%); 14325/42027 objects misplaced (34.085%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:23:50.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:50.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2591/44274 objects degraded (5.852%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[107399]: osdmap e366: 8 total, 7 up, 6 in 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[107399]: pgmap v1274: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 53 active+clean; 874 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 2397/42027 objects degraded (5.703%); 14325/42027 objects misplaced (34.085%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2591/44274 objects degraded (5.852%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[112165]: osdmap e366: 8 total, 7 up, 6 in 2024-09-14T04:23:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:49 smithi001 ceph-mon[112165]: pgmap v1274: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 53 active+clean; 874 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 2397/42027 objects degraded (5.703%); 14325/42027 objects misplaced (34.085%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:23:50.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:50.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:50.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:50.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:50.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:50.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:50.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:50.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:50.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:50.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:50.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:50.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:50.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:50.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:50.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:51.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:51.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:51.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:51.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:51.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[107399]: osdmap e367: 8 total, 7 up, 6 in 2024-09-14T04:23:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:50 smithi001 ceph-mon[112165]: osdmap e367: 8 total, 7 up, 6 in 2024-09-14T04:23:51.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:51.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:23:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:50 smithi012 ceph-mon[101180]: osdmap e367: 8 total, 7 up, 6 in 2024-09-14T04:23:51.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:51.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:51.504 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:51.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:51.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:51.691 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:51.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:51.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:51.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:51.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:51.966 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:52.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:52.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:52.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:52.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:52.202 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:23:52.202 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:23:52.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:52.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:52.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:52 smithi001 ceph-mon[107399]: osdmap e368: 8 total, 7 up, 6 in 2024-09-14T04:23:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:52 smithi001 ceph-mon[107399]: pgmap v1277: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 55 active+clean; 827 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 2541/39732 objects degraded (6.395%); 13814/39732 objects misplaced (34.768%); 7.4 MiB/s, 119 objects/s recovering 2024-09-14T04:23:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:52 smithi001 ceph-mon[112165]: osdmap e368: 8 total, 7 up, 6 in 2024-09-14T04:23:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:52 smithi001 ceph-mon[112165]: pgmap v1277: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 55 active+clean; 827 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 2541/39732 objects degraded (6.395%); 13814/39732 objects misplaced (34.768%); 7.4 MiB/s, 119 objects/s recovering 2024-09-14T04:23:52.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:52.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:52.443 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:52.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:52.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:52 smithi012 ceph-mon[101180]: osdmap e368: 8 total, 7 up, 6 in 2024-09-14T04:23:52.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:52 smithi012 ceph-mon[101180]: pgmap v1277: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 peering, 1 unknown, 1 active+undersized+degraded+remapped, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 55 active+clean; 827 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 2541/39732 objects degraded (6.395%); 13814/39732 objects misplaced (34.768%); 7.4 MiB/s, 119 objects/s recovering 2024-09-14T04:23:52.607 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:52.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:52.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:52.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:52.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:52.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:52.927 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [5] live_osds: [1, 3, 4, 6, 7, 0, 2] 2024-09-14T04:23:52.927 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-09-14T04:23:52.928 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-09-14T04:23:52.928 DEBUG:teuthology.orchestra.run.smithi012:osd.5> sudo journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5.service 2024-09-14T04:23:52.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl start ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5 2024-09-14T04:23:52.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:53.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:53.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:53.148 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:53.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:53.266 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-mon[101180]: osdmap e369: 8 total, 7 up, 6 in 2024-09-14T04:23:53.266 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 systemd[1]: Starting Ceph osd.5 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:23:53.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:53.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:53.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:53.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:53 smithi001 ceph-mon[107399]: osdmap e369: 8 total, 7 up, 6 in 2024-09-14T04:23:53.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:53 smithi001 ceph-mon[112165]: osdmap e369: 8 total, 7 up, 6 in 2024-09-14T04:23:53.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:53.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:53.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:53.531 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.167870109 +0000 UTC m=+0.019584035 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:23:53.531 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.294143765 +0000 UTC m=+0.145857690 container create dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:23:53.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:53.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:53.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:53.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:53.846 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.530384639 +0000 UTC m=+0.382098580 container init dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T04:23:53.846 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.536056632 +0000 UTC m=+0.387770559 container start dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b) 2024-09-14T04:23:53.846 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.664104683 +0000 UTC m=+0.515818618 container attach dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-14T04:23:53.903 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:53.934 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:53.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:54.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:54.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:54.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T04:23:54.201 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate[209989]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 bash[209974]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T04:23:54.202 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:53 smithi012 podman[209974]: 2024-09-14 04:23:53.977156327 +0000 UTC m=+0.828870261 container died dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:23:54.208 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:54.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:54 smithi001 ceph-mon[107399]: pgmap v1279: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 741 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 1978/35634 objects degraded (5.551%); 12002/35634 objects misplaced (33.681%); 15 MiB/s, 234 objects/s recovering 2024-09-14T04:23:54.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 2541/39732 objects degraded (6.395%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:54 smithi001 ceph-mon[112165]: pgmap v1279: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 741 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 1978/35634 objects degraded (5.551%); 12002/35634 objects misplaced (33.681%); 15 MiB/s, 234 objects/s recovering 2024-09-14T04:23:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 2541/39732 objects degraded (6.395%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:54.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:54.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:54.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:54.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:54.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:54.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:54 smithi012 ceph-mon[101180]: pgmap v1279: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 741 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 1978/35634 objects degraded (5.551%); 12002/35634 objects misplaced (33.681%); 15 MiB/s, 234 objects/s recovering 2024-09-14T04:23:54.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 2541/39732 objects degraded (6.395%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:54.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:54.617 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:23:54.650 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:54.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:54.734 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:54.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:54.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:54.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:54.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:54.913 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:54 smithi012 podman[210138]: 2024-09-14 04:23:54.661494172 +0000 UTC m=+0.677959789 container remove dad794aaa7e473260a0169b2d9566f16a896500fd7de918ec500aa5aefa4c8f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:23:54.913 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:54 smithi012 podman[210255]: 2024-09-14 04:23:54.786393437 +0000 UTC m=+0.020407205 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:23:54.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:55.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:55.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:55.184 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:54 smithi012 podman[210255]: 2024-09-14 04:23:54.912796019 +0000 UTC m=+0.146809768 container create f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-09-14T04:23:55.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:55.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:55.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:23:55.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:55.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:55.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:55 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T04:23:55.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:55 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T04:23:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.498 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:55.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:55.575 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:23:55.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:55 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2024-09-14T04:23:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 podman[210255]: 2024-09-14 04:23:55.182924725 +0000 UTC m=+0.416938479 container init f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:23:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 podman[210255]: 2024-09-14 04:23:55.189755227 +0000 UTC m=+0.423768968 container start f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T04:23:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 ceph-osd[210293]: -- 172.21.15.12:0/2607102069 <== mon.1 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1fd6f71e0 con 0x55f1fd86a000 2024-09-14T04:23:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 bash[210255]: f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 2024-09-14T04:23:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 systemd[1]: Started Ceph osd.5 for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:23:55.599 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:55.626 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:55.728 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:55.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:55.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:55.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:55.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:56.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:56.012 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:23:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:23:55.808+0000 7f79afc56640 -1 Falling back to public interface 2024-09-14T04:23:56.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:56.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:56.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:56.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:56.259 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:23:56.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:56.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:56 smithi012 ceph-mon[101180]: pgmap v1280: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 685 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 1811/32928 objects degraded (5.500%); 10921/32928 objects misplaced (33.166%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:23:56.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:56.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:56.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:56 smithi001 ceph-mon[112165]: pgmap v1280: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 685 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 1811/32928 objects degraded (5.500%); 10921/32928 objects misplaced (33.166%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:23:56.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:56 smithi001 ceph-mon[107399]: pgmap v1280: 119 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 685 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 1811/32928 objects degraded (5.500%); 10921/32928 objects misplaced (33.166%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:23:56.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:56.577 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:56.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:56.718 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:56.754 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:23:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:23:56] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.51.0" 2024-09-14T04:23:56.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:56.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:56.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:56.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:56.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:56.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:57.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:57.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:57.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:57.228 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:57.241 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:23:57.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4048257164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:57 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:57.381 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4048257164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.381 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:57 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4048257164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.381 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:57 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:23:57.459 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:57.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:57.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:57.641 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:57.743 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:57.775 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:57.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:57.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:57.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:58.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:58.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:58.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:58.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:58.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:58.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:58.301 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[107399]: osdmap e370: 8 total, 7 up, 6 in 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[107399]: pgmap v1282: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 524 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 1470/25191 objects degraded (5.835%); 9859/25191 objects misplaced (39.137%); 7.5 MiB/s, 118 objects/s recovering 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[112165]: osdmap e370: 8 total, 7 up, 6 in 2024-09-14T04:23:58.302 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:58 smithi001 ceph-mon[112165]: pgmap v1282: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 524 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 1470/25191 objects degraded (5.835%); 9859/25191 objects misplaced (39.137%); 7.5 MiB/s, 118 objects/s recovering 2024-09-14T04:23:58.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:23:58.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:23:58.486 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:58 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:23:58.486 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:58 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:23:58.486 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:58 smithi012 ceph-mon[101180]: osdmap e370: 8 total, 7 up, 6 in 2024-09-14T04:23:58.486 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:58 smithi012 ceph-mon[101180]: pgmap v1282: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 524 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 1470/25191 objects degraded (5.835%); 9859/25191 objects misplaced (39.137%); 7.5 MiB/s, 118 objects/s recovering 2024-09-14T04:23:58.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:58.564 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:58.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:58.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:58.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:no valid command found; 10 closest matches: 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:0 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:1 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:2 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:abort 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:assert 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs files list 2024-09-14T04:23:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs stats 2024-09-14T04:23:58.771 INFO:teuthology.orchestra.run.smithi012.stderr:bluestore bluefs device info [] 2024-09-14T04:23:58.771 INFO:teuthology.orchestra.run.smithi012.stderr:config diff 2024-09-14T04:23:58.771 INFO:teuthology.orchestra.run.smithi012.stderr:admin_socket: invalid command 2024-09-14T04:23:58.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:58.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:58.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:58.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:23:58.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:58.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:23:59.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:59.092 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:59.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:23:59.222 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:23:59.324 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:59.361 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:59.429 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:23:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 1470/25191 objects degraded (5.835%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:23:59.461 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-09-14T04:23:59.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:23:59.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:23:59.589 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:23:59.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:23:59.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:23:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 1470/25191 objects degraded (5.835%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:23:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 1470/25191 objects degraded (5.835%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:23:59.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:23:59.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:23:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:23:59.854 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:23:59.887 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:23:59.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:23:59.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:23:59.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.020 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:00.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:00.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:00.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:00.264 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:00.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:00.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:00.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:00.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:00.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:00 smithi012 ceph-mon[101180]: pgmap v1283: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 457 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 739 op/s; 1189/21969 objects degraded (5.412%); 8542/21969 objects misplaced (38.882%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:24:00.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:00.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:00 smithi001 ceph-mon[107399]: pgmap v1283: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 457 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 739 op/s; 1189/21969 objects degraded (5.412%); 8542/21969 objects misplaced (38.882%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:24:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:00 smithi001 ceph-mon[112165]: pgmap v1283: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 457 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 739 op/s; 1189/21969 objects degraded (5.412%); 8542/21969 objects misplaced (38.882%); 5.8 MiB/s, 93 objects/s recovering 2024-09-14T04:24:00.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:00.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:00.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:00.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:00.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:01.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:01.058 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:01.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.208 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:01.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:01.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:01.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:01.358 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:24:01.358 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:24:01.394 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:24:01.394 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:24:01.416 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2615137193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:01.417 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2615137193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:01.425 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:24:01.425 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:24:01.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:01.465 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:24:01.465 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:24:01.484 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:01.500 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:24:01.500 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:24:01.501 INFO:teuthology.orchestra.run.smithi001.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-14T04:24:01.536 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:24:01.537 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:24:01.568 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:24:01.568 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:24:01.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:01.612 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:01.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:01.802 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:01.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2615137193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:01.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:01.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:01.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:02.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:02.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:02.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:02.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:02.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:02.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:02.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:02.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:02.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:02.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:02.711 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:02.717 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:02 smithi001 ceph-mon[107399]: pgmap v1284: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 408 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1025/19656 objects degraded (5.215%); 8081/19656 objects misplaced (41.112%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:24:02.717 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:02 smithi001 ceph-mon[112165]: pgmap v1284: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 408 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1025/19656 objects degraded (5.215%); 8081/19656 objects misplaced (41.112%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:24:02.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:02 smithi012 ceph-mon[101180]: pgmap v1284: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 408 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 573 op/s; 1025/19656 objects degraded (5.215%); 8081/19656 objects misplaced (41.112%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:24:02.849 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:02.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:02.889 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:24:02.889 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:24:02.889 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:24:02.889 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:24:02.889 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:24:02.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:02.976 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:02.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:03.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:03.102 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:03.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:03.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:03.242 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:03.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:24:03.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:24:03.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:03.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:03.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:03.512 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:03.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:03.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:03.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:03.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:03.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:03.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:03.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:04.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:24:04.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:04.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:24:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:04.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:04.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:04.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:04.357 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:04.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:04.457 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:04.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:24:04.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:04.577 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:04.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:04.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:04.734 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:24:04.789 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:04 smithi001 ceph-mon[107399]: pgmap v1285: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 268 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 756 op/s; 755/12933 objects degraded (5.838%); 6271/12933 objects misplaced (48.488%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:04.789 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 755/12933 objects degraded (5.838%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:04.789 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:04 smithi001 ceph-mon[112165]: pgmap v1285: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 268 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 756 op/s; 755/12933 objects degraded (5.838%); 6271/12933 objects misplaced (48.488%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:04.789 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 755/12933 objects degraded (5.838%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:04.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:04 smithi012 ceph-mon[101180]: pgmap v1285: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 268 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 756 op/s; 755/12933 objects degraded (5.838%); 6271/12933 objects misplaced (48.488%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 755/12933 objects degraded (5.838%), 8 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:04.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:04.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:04.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:05.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:05.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:05.112 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:05.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:05.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:05.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:05.303 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:05.325 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:24:05.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:05.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:05.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:05.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:05.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:24:05.443+0000 7f79afc56640 -1 osd.5 313 log_to_monitors true 2024-09-14T04:24:05.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:05.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:05.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:05.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:05.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:05.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:06.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:06.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 145657 objects 2024-09-14T04:24:06.092 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :256.54 2024-09-14T04:24:06.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:06.148 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:06.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:06.198 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:24:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:24:06.073+0000 7f79a7a00640 -1 osd.5 371 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:24:06.198 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:06.198 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:06.198 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-mon[101180]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.198 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-mon[101180]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.198 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:05 smithi012 ceph-mon[101180]: pgmap v1286: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 191 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 508/9198 objects degraded (5.523%); 4786/9198 objects misplaced (52.033%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[107399]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[107399]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[107399]: pgmap v1286: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 191 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 508/9198 objects degraded (5.523%); 4786/9198 objects misplaced (52.033%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[112165]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[112165]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T04:24:06.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:05 smithi001 ceph-mon[112165]: pgmap v1286: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 191 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 508/9198 objects degraded (5.523%); 4786/9198 objects misplaced (52.033%); 2.8 MiB/s, 45 objects/s recovering 2024-09-14T04:24:06.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:06.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:06.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:06.399 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:06.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:06.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:06.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:06.556 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:24:06.556 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:24:06.556 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2024-09-14T04:24:06.556 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:24:06.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:06.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:06.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:06.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:06.772 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:06.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:06.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:06.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:06] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-09-14T04:24:06.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:07.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:06 smithi012 ceph-mon[101180]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T04:24:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:06 smithi012 ceph-mon[101180]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:06 smithi012 ceph-mon[101180]: osdmap e371: 8 total, 7 up, 6 in 2024-09-14T04:24:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:06 smithi012 ceph-mon[101180]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.120 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:24:07.124 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:07.185 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:07.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:07.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:07.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:07.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:07.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[112165]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[112165]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[112165]: osdmap e371: 8 total, 7 up, 6 in 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[112165]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[107399]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[107399]: from='osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[107399]: osdmap e371: 8 total, 7 up, 6 in 2024-09-14T04:24:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:06 smithi001 ceph-mon[107399]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-14T04:24:07.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:07.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:07.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:07.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:07.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:07.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:07.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:07.944 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:07.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:08.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:08.072 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:08.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.197 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:08.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:08.235 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616] boot 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: osdmap e372: 8 total, 8 up, 6 in 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: pgmap v1289: 119 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 16 active+clean+remapped, 11 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 34 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 131/1662 objects degraded (7.882%); 2969/1662 objects misplaced (178.640%); 3.5 MiB/s, 96 objects/s recovering 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[112165]: osdmap e373: 8 total, 8 up, 6 in 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616] boot 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: osdmap e372: 8 total, 8 up, 6 in 2024-09-14T04:24:08.236 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T04:24:08.237 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: pgmap v1289: 119 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 16 active+clean+remapped, 11 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 34 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 131/1662 objects degraded (7.882%); 2969/1662 objects misplaced (178.640%); 3.5 MiB/s, 96 objects/s recovering 2024-09-14T04:24:08.237 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:07 smithi001 ceph-mon[107399]: osdmap e373: 8 total, 8 up, 6 in 2024-09-14T04:24:08.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:08.321 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: osd.5 [v2:172.21.15.12:6808/3238411616,v1:172.21.15.12:6809/3238411616] boot 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: osdmap e372: 8 total, 8 up, 6 in 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: pgmap v1289: 119 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 16 active+clean+remapped, 11 active+remapped+backfill_wait, 16 active+undersized, 4 active+undersized+degraded, 60 active+clean; 34 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 131/1662 objects degraded (7.882%); 2969/1662 objects misplaced (178.640%); 3.5 MiB/s, 96 objects/s recovering 2024-09-14T04:24:08.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:07 smithi012 ceph-mon[101180]: osdmap e373: 8 total, 8 up, 6 in 2024-09-14T04:24:08.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:08.447 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:08.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:08.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:08.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:08.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:08.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.833 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:08.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:08.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:08.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:08.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:09.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:09.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:09.093 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.0 continuing backfill to osd.3 from (301'10149,371'16815] MIN to 371'16815 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16815 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.16 continuing backfill to osd.1 from (301'10913,371'17066] MIN to 371'17066 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17066 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.10 continuing backfill to osd.3 from (301'10149,371'16745] MIN to 371'16745 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.10 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16745 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.13 continuing backfill to osd.1 from (301'10421,371'16587] MIN to 371'16587 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.13 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16587 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.3 continuing backfill to osd.1 from (301'10421,371'16613] MIN to 371'16613 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16613 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.6 continuing backfill to osd.1 from (301'10913,371'17153] MIN to 371'17153 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17153 2024-09-14T04:24:09.094 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[107399]: osdmap e374: 8 total, 8 up, 6 in 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.0 continuing backfill to osd.3 from (301'10149,371'16815] MIN to 371'16815 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16815 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.16 continuing backfill to osd.1 from (301'10913,371'17066] MIN to 371'17066 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17066 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.10 continuing backfill to osd.3 from (301'10149,371'16745] MIN to 371'16745 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.10 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16745 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.13 continuing backfill to osd.1 from (301'10421,371'16587] MIN to 371'16587 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.13 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16587 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.3 continuing backfill to osd.1 from (301'10421,371'16613] MIN to 371'16613 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16613 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.6 continuing backfill to osd.1 from (301'10913,371'17153] MIN to 371'17153 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17153 2024-09-14T04:24:09.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:08 smithi001 ceph-mon[112165]: osdmap e374: 8 total, 8 up, 6 in 2024-09-14T04:24:09.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:09.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:09.232 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111750 11.6 11:6780736c:::benchmark_data_smithi001_228075_object111749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 516.80196949100002, 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0279308629999999, 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.233 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111750 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.234 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427647+0000", 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427649+0000", 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8339999999999999e-06 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.235 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427674+0000", 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4975000000000001e-05 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427687+0000", 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3276e-05 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.427697+0000", 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3239999999999992e-06 2024-09-14T04:24:09.236 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431409+0000", 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037127940000000002 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.431421+0000", 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1535000000000001e-05 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.237 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438346+0000", 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.006924989 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438476+0000", 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000129815 2024-09-14T04:24:09.238 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.438613+0000", 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013711100000000001 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.239 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447342+0000", 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087294480000000008 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455412+0000", 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080694630000000007 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.240 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455479+0000", 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7631000000000004e-05 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455578+0000", 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8667999999999998e-05 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111764 11.6 11:66131855:::benchmark_data_smithi001_228075_object111763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:24:09.241 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 516.77491882000004, 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0049983549999999, 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111764 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.242 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454698+0000", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454700+0000", 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.046e-06 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.243 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454723+0000", 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3271000000000001e-05 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454741+0000", 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8281999999999999e-05 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.244 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454749+0000", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4370000000000001e-06 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454812+0000", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3596000000000006e-05 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.454823+0000", 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1090000000000001e-05 2024-09-14T04:24:09.245 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.442746+0000", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98792279299999997 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.442898+0000", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015215300000000001 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.443068+0000", 2024-09-14T04:24:09.246 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000169958 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.455636+0000", 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012567807 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459433+0000", 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037966279999999998 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.247 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459607+0000", 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017445899999999999 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459696+0000", 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8834999999999999e-05 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.248 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.35451.0:111766 11.6 11:61ce0161:::benchmark_data_smithi001_228075_object111765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 516.77365085700001, 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0098253399999999, 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.249 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.35451", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/1943504714", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 111766 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.250 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455966+0000", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455967+0000", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1629999999999999e-06 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.455988+0000", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1546999999999999e-05 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.251 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456002+0000", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3473e-05 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456008+0000", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8939999999999998e-06 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456061+0000", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2991999999999997e-05 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.252 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:32.456071+0000", 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0327e-05 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447464+0000", 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.991392618 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447619+0000", 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000154878 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.253 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.447781+0000", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000162826 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.459705+0000", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011923173 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465640+0000", 2024-09-14T04:24:09.254 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059350999999999996 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465701+0000", 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0872000000000003e-05 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:15:33.465791+0000", 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0476999999999994e-05 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.255 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145645 12.c 12:3996658d:::benchmark_data_smithi001_255951_object145644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 262.78252777199998, 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5001031949999999, 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.256 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145645 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.257 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447090+0000", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5370000000000001e-06 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447116+0000", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5673999999999998e-05 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447128+0000", 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1878e-05 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.258 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447136+0000", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0989999999999999e-06 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.451501+0000", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004364723 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.451511+0000", 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0339999999999999e-05 2024-09-14T04:24:09.259 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.436184+0000", 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98467291999999995 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.436197+0000", 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2951e-05 2024-09-14T04:24:09.260 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940260+0000", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.504063127 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940396+0000", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135499 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940581+0000", 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018567799999999999 2024-09-14T04:24:09.261 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944744+0000", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041623989999999998 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.946037+0000", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012937890000000001 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947043+0000", 2024-09-14T04:24:09.262 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001005508 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947101+0000", 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8458000000000002e-05 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947192+0000", 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0614999999999999e-05 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.263 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145621 12.a 12:5542fd65:::benchmark_data_smithi001_255951_object145620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 261.78859655000002, 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4993388519999999, 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.264 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145621 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.265 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441022+0000", 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6819999999999999e-06 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.266 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441434+0000", 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00041247899999999999 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441451+0000", 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6762e-05 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441460+0000", 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8799999999999997e-06 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.267 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441708+0000", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024832400000000002 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441719+0000", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0411000000000001e-05 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940283+0000", 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4985643479999999 2024-09-14T04:24:09.268 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940359+0000", 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5965999999999997e-05 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145627 12.a 12:5df13aaa:::benchmark_data_smithi001_255951_object145626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 261.78813790499999, 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.499140605, 2024-09-14T04:24:09.269 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145627 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.270 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3799999999999996e-07 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441541+0000", 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1072000000000001e-05 2024-09-14T04:24:09.271 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441549+0000", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4489999999999999e-06 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441555+0000", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1689999999999997e-06 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441864+0000", 2024-09-14T04:24:09.272 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030918299999999998 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441872+0000", 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1110000000000008e-06 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940553+0000", 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.498680727 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.273 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940619+0000", 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6155999999999995e-05 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145651 12.a 12:5b322833:::benchmark_data_smithi001_255951_object145650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 261.78805289600001, 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5069970850000001, 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.274 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145651 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.275 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1699999999999998e-07 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441611+0000", 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6927e-05 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.276 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441620+0000", 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2110000000000004e-06 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441627+0000", 2024-09-14T04:24:09.277 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2230000000000004e-06 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441897+0000", 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027031800000000002 2024-09-14T04:24:09.278 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441904+0000", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7399999999999998e-06 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940646+0000", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.498742332 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940762+0000", 2024-09-14T04:24:09.279 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115798 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940921+0000", 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015967999999999999 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944358+0000", 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034363340000000001 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.280 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.946696+0000", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002338082 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948409+0000", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0017135200000000001 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948463+0000", 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4008999999999997e-05 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.281 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948561+0000", 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7393999999999996e-05 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243554 12.3 12:c3bf76db:::benchmark_data_smithi001_255951_object97894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 118.446002243, 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7736291980000001, 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.282 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243554 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.283 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783616+0000", 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0080000000000001e-06 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.284 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783624+0000", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1389999999999998e-06 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783641+0000", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7342000000000001e-05 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783648+0000", 2024-09-14T04:24:09.285 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0720000000000002e-06 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783722+0000", 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4325000000000004e-05 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783737+0000", 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4562000000000001e-05 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.286 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313440+0000", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.52970318299999997 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313452+0000", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2206e-05 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.537509+0000", 2024-09-14T04:24:09.287 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2240570070000001 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.537645+0000", 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000136141 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.538767+0000", 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001121976 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.288 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.547961+0000", 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0091935539999999996 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.550934+0000", 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029733049999999999 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554104+0000", 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00316972 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.289 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557038+0000", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029345489999999998 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557110+0000", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1348999999999997e-05 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557244+0000", 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013375999999999999 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.290 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243561 12.3 12:c5ad174e:::benchmark_data_smithi001_255951_object97901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 118.050378145, 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3783672410000001, 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.291 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243561 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.292 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179241+0000", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2689999999999998e-06 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179256+0000", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5534000000000001e-05 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179279+0000", 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2751e-05 2024-09-14T04:24:09.293 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179290+0000", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0593e-05 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179365+0000", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5424000000000002e-05 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179374+0000", 2024-09-14T04:24:09.294 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3139999999999993e-06 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313477+0000", 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13410250800000001 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313485+0000", 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8169999999999999e-06 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.295 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.547996+0000", 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2345118159999999 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.548122+0000", 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000125764 2024-09-14T04:24:09.296 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.548263+0000", 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014107900000000001 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.297 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554033+0000", 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057697720000000003 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.298 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554076+0000", 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2695999999999999e-05 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554110+0000", 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4564999999999999e-05 2024-09-14T04:24:09.299 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.300 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.300 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.300 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557438+0000", 2024-09-14T04:24:09.300 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00332775 2024-09-14T04:24:09.300 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557494+0000", 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6064999999999999e-05 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.301 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557606+0000", 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000111524 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.302 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243562 12.3 12:c85133a2:::benchmark_data_smithi001_255951_object97902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.95520123599999, 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.283526199, 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243562 2024-09-14T04:24:09.303 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.304 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.305 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274418+0000", 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2330000000000001e-06 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.306 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274426+0000", 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6449999999999994e-06 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274448+0000", 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1542000000000001e-05 2024-09-14T04:24:09.307 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274458+0000", 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.308 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274542+0000", 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3781000000000003e-05 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.309 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274555+0000", 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2846e-05 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.310 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313507+0000", 2024-09-14T04:24:09.311 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.038952458000000002 2024-09-14T04:24:09.311 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.311 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.311 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.311 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313515+0000", 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.550967+0000", 2024-09-14T04:24:09.312 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2374517810000001 2024-09-14T04:24:09.313 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.313 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.313 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.313 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.551087+0000", 2024-09-14T04:24:09.313 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011969 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.551225+0000", 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013870899999999999 2024-09-14T04:24:09.314 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554116+0000", 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028902979999999999 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.315 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557298+0000", 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031826139999999998 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.316 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557347+0000", 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9183e-05 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.317 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557664+0000", 2024-09-14T04:24:09.318 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000316466 2024-09-14T04:24:09.318 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.318 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.318 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.318 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557722+0000", 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8576999999999998e-05 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557942+0000", 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021917900000000001 2024-09-14T04:24:09.319 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243568 12.e 12:7f8cd4cb:::benchmark_data_smithi001_255951_object97908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:24:09.320 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.84493751399999, 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.245202098, 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.321 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243568 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.322 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384680+0000", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4100000000000001e-06 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384684+0000", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9720000000000003e-06 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384696+0000", 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1674999999999999e-05 2024-09-14T04:24:09.323 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384702+0000", 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9660000000000001e-06 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384764+0000", 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2053999999999999e-05 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.324 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384778+0000", 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3574000000000001e-05 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560158+0000", 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.175380281 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560276+0000", 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117953 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.325 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560747+0000", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00047152199999999999 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626865+0000", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.066117457000000004 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626930+0000", 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5208000000000003e-05 2024-09-14T04:24:09.326 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626980+0000", 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9792e-05 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629676+0000", 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002696155 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.327 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629740+0000", 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4349000000000003e-05 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629881+0000", 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014072999999999999 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243569 12.e 12:72fca126:::benchmark_data_smithi001_255951_object97909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:24:09.328 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 117.842665271, 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2435670029999999, 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.329 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243569 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.330 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386952+0000", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1349999999999999e-06 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386957+0000", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3080000000000001e-06 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386967+0000", 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0412e-05 2024-09-14T04:24:09.331 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386974+0000", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3509999999999998e-06 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.387029+0000", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5467999999999998e-05 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.387036+0000", 2024-09-14T04:24:09.332 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7089999999999996e-06 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627054+0000", 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.240018735 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627181+0000", 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012620700000000001 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.333 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627317+0000", 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013682200000000001 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629892+0000", 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025746620000000001 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.334 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629931+0000", 2024-09-14T04:24:09.335 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8968999999999999e-05 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630135+0000", 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000203799 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630243+0000", 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108099 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.336 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630305+0000", 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1724999999999996e-05 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630518+0000", 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021360199999999999 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.337 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254264 12.16 12:6e91794d:::benchmark_data_smithi001_255951_object108604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.637909715999996, 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.075071672, 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254264 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.338 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:24:09.339 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.340 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.340 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.340 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.340 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591709+0000", 2024-09-14T04:24:09.340 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8759999999999999e-06 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591717+0000", 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9489999999999996e-06 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591736+0000", 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9528999999999999e-05 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.341 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591745+0000", 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6300000000000004e-06 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591841+0000", 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6379000000000007e-05 2024-09-14T04:24:09.342 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591857+0000", 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6163000000000001e-05 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.343 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.663671+0000", 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0718133569999999 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.663838+0000", 2024-09-14T04:24:09.344 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000167651 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.664192+0000", 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035332499999999998 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.345 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666120+0000", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001928443 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666342+0000", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022153200000000001 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666418+0000", 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5950000000000003e-05 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.346 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666483+0000", 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5798999999999998e-05 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666553+0000", 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9492000000000005e-05 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.347 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666629+0000", 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6198000000000005e-05 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666779+0000", 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014939900000000001 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.348 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254293 12.6 12:60fdceba:::benchmark_data_smithi001_255951_object108633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.543786257999997, 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0043203650000001, 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254293 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.349 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685832+0000", 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.296e-06 2024-09-14T04:24:09.350 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685836+0000", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4320000000000001e-06 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685849+0000", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2768e-05 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685854+0000", 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0640000000000001e-06 2024-09-14T04:24:09.351 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685999+0000", 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014539800000000001 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.686009+0000", 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0192e-05 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.352 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676321+0000", 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99031173400000005 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676450+0000", 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012866599999999999 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.353 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676770+0000", 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000320501 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681206+0000", 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044353329999999996 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681282+0000", 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6676999999999998e-05 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.354 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687652+0000", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063695410000000003 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687722+0000", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0320000000000004e-05 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689967+0000", 2024-09-14T04:24:09.355 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022448670000000002 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.690027+0000", 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.029e-05 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.690151+0000", 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012328600000000001 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.356 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.357 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.357 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.360 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.360 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254316 12.16 12:6a29b3f6:::benchmark_data_smithi001_255951_object108656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.360 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:24:09.360 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.504532144999999, 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94999470500000005, 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254316 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725086+0000", 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2780000000000001e-06 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725090+0000", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3069999999999996e-06 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725102+0000", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1569999999999999e-05 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725108+0000", 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.083e-06 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725171+0000", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3691999999999999e-05 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725185+0000", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3451e-05 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666839+0000", 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94165448399999996 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666979+0000", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013989499999999999 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.667286+0000", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000306674 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670323+0000", 2024-09-14T04:24:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003037414 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670371+0000", 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7908000000000003e-05 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674661+0000", 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042897500000000002 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674841+0000", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000179617 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674883+0000", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2491999999999999e-05 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674944+0000", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1408000000000006e-05 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675079+0000", 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013468200000000001 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254322 12.6 12:672ca547:::benchmark_data_smithi001_255951_object108662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.489917782000006, 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95668792300000005, 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254322 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739700+0000", 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5099999999999999e-07 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739703+0000", 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4680000000000001e-06 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739712+0000", 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4989999999999994e-06 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739716+0000", 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6800000000000001e-06 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739768+0000", 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2181000000000001e-05 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739779+0000", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0188e-05 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681316+0000", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94153731699999998 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681452+0000", 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001359 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681774+0000", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032192599999999999 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687571+0000", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057971990000000003 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689807+0000", 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022359540000000001 2024-09-14T04:24:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.692137+0000", 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023296929999999999 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.694001+0000", 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001864276 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696206+0000", 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022052460000000001 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696262+0000", 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5491000000000001e-05 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696387+0000", 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012515399999999999 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254327 12.6 12:63c13416:::benchmark_data_smithi001_255951_object108667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.478884844999996, 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94596290800000005, 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254327 2024-09-14T04:24:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750734+0000", 2024-09-14T04:24:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7630000000000001e-06 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750740+0000", 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7669999999999998e-06 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750757+0000", 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6484999999999999e-05 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750765+0000", 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0520000000000002e-06 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750830+0000", 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5215000000000005e-05 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750844+0000", 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3781000000000001e-05 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.683972+0000", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93312859599999998 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.684092+0000", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001199 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.684372+0000", 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000279354 2024-09-14T04:24:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687687+0000", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033157579999999998 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689860+0000", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021729589999999999 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.692143+0000", 2024-09-14T04:24:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022829959999999998 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.694045+0000", 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001901944 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696493+0000", 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024481490000000002 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696556+0000", 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2799999999999995e-05 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696695+0000", 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013838899999999999 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254363 12.16 12:6b264e56:::benchmark_data_smithi001_255951_object108703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.389914234000003, 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.83572432600000002, 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254363 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839704+0000", 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.578e-06 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839709+0000", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6129999999999997e-06 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839722+0000", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.348e-05 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839730+0000", 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.405e-06 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839802+0000", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1544999999999993e-05 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839810+0000", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8480000000000001e-06 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.668751+0000", 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.82894146000000002 2024-09-14T04:24:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.668941+0000", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018943000000000001 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.669225+0000", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00028416599999999998 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674646+0000", 2024-09-14T04:24:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005421131 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674668+0000", 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1801999999999999e-05 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674786+0000", 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011822799999999999 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675213+0000", 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00042697699999999999 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675255+0000", 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1730999999999997e-05 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675308+0000", 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3114e-05 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675427+0000", 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118818 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254376 12.16 12:684dc283:::benchmark_data_smithi001_255951_object108716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.372586890999997, 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.81920720899999999, 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254376 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857031+0000", 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1599999999999999e-06 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857034+0000", 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5889999999999999e-06 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857047+0000", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2252000000000001e-05 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857053+0000", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6019999999999996e-06 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857122+0000", 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8534999999999993e-05 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857135+0000", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3030000000000001e-05 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670546+0000", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.81341160700000004 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670687+0000", 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014010800000000001 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.671031+0000", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034398700000000003 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674673+0000", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036429240000000001 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675122+0000", 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00044859499999999998 2024-09-14T04:24:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675165+0000", 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3022999999999998e-05 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675510+0000", 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034440300000000002 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675550+0000", 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0927999999999998e-05 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675602+0000", 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1935000000000003e-05 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676237+0000", 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00063453100000000003 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254417 12.16 12:6fb580d5:::benchmark_data_smithi001_255951_object108757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:05.030869+0000", 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 64.198748096000003, 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.650295489, 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254417 2024-09-14T04:24:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030869+0000", 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030869+0000", 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030870+0000", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.283e-06 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030874+0000", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.048e-06 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030886+0000", 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1671999999999999e-05 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030894+0000", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0169999999999996e-06 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030948+0000", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4598999999999997e-05 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.030956+0000", 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1769999999999998e-06 2024-09-14T04:24:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.672542+0000", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.64158575799999995 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.672678+0000", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001362 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.673019+0000", 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034040200000000001 2024-09-14T04:24:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675468+0000", 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024494209999999998 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676290+0000", 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000821805 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.680879+0000", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045891910000000003 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.680918+0000", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9239999999999997e-05 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.680958+0000", 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0098000000000001e-05 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681019+0000", 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0667e-05 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681164+0000", 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014491100000000001 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:24:09.406 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:24:09.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.0 continuing backfill to osd.3 from (301'10149,371'16815] MIN to 371'16815 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16815 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.16 continuing backfill to osd.1 from (301'10913,371'17066] MIN to 371'17066 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17066 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.10 continuing backfill to osd.3 from (301'10149,371'16745] MIN to 371'16745 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.10 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16745 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.13 continuing backfill to osd.1 from (301'10421,371'16587] MIN to 371'16587 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.13 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16587 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.3 continuing backfill to osd.1 from (301'10421,371'16613] MIN to 371'16613 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'16613 2024-09-14T04:24:09.408 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.6 continuing backfill to osd.1 from (301'10913,371'17153] MIN to 371'17153 2024-09-14T04:24:09.409 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 371'17153 2024-09-14T04:24:09.409 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:08 smithi012 ceph-mon[101180]: osdmap e374: 8 total, 8 up, 6 in 2024-09-14T04:24:09.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:09.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:09.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:09.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:09.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:09.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:09.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:09.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:09.910 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:09.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:09.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:09.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:10.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:10.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:09 smithi012 ceph-mon[101180]: 12.11 continuing backfill to osd.1 from (301'10980,371'16836] MIN to 371'16836 2024-09-14T04:24:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:09 smithi012 ceph-mon[101180]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 131/1662 objects degraded (7.882%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:09 smithi012 ceph-mon[101180]: osdmap e375: 8 total, 8 up, 6 in 2024-09-14T04:24:10.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:09 smithi012 ceph-mon[101180]: pgmap v1293: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+clean+remapped, 11 active+remapped+backfill_wait, 15 active+undersized, 3 active+undersized+degraded, 67 active+clean; 11 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 28/585 objects degraded (4.786%); 2549/585 objects misplaced (435.726%); 3.1 MiB/s, 153 objects/s recovering 2024-09-14T04:24:10.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.195 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:10.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:10.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:10.318 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:10.395 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[112165]: 12.11 continuing backfill to osd.1 from (301'10980,371'16836] MIN to 371'16836 2024-09-14T04:24:10.396 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[112165]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:10.396 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 131/1662 objects degraded (7.882%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:10.396 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[112165]: osdmap e375: 8 total, 8 up, 6 in 2024-09-14T04:24:10.396 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[112165]: pgmap v1293: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+clean+remapped, 11 active+remapped+backfill_wait, 15 active+undersized, 3 active+undersized+degraded, 67 active+clean; 11 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 28/585 objects degraded (4.786%); 2549/585 objects misplaced (435.726%); 3.1 MiB/s, 153 objects/s recovering 2024-09-14T04:24:10.396 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[107399]: 12.11 continuing backfill to osd.1 from (301'10980,371'16836] MIN to 371'16836 2024-09-14T04:24:10.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[107399]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:10.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 131/1662 objects degraded (7.882%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:10.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[107399]: osdmap e375: 8 total, 8 up, 6 in 2024-09-14T04:24:10.397 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:09 smithi001 ceph-mon[107399]: pgmap v1293: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 8 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+clean+remapped, 11 active+remapped+backfill_wait, 15 active+undersized, 3 active+undersized+degraded, 67 active+clean; 11 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 28/585 objects degraded (4.786%); 2549/585 objects misplaced (435.726%); 3.1 MiB/s, 153 objects/s recovering 2024-09-14T04:24:10.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:10.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:10.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:10.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:10.571 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:10.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:10.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:10.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:10.822 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:10.924 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:10.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:10.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:11.050 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:11.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:11.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:11.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:11.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:11.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:11.380 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:11 smithi001 ceph-mon[107399]: osdmap e376: 8 total, 8 up, 6 in 2024-09-14T04:24:11.380 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:11 smithi001 ceph-mon[112165]: osdmap e376: 8 total, 8 up, 6 in 2024-09-14T04:24:11.419 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2024-09-14T04:24:11.419 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2024-09-14T04:24:11.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:11.489 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:11.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:11.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:11.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:11 smithi012 ceph-mon[101180]: osdmap e376: 8 total, 8 up, 6 in 2024-09-14T04:24:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.614 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:11.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:11.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:11.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:11.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:11.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:11.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:11.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:11.975 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:12.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:12.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:12.128 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:12.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:12.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:12.357 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:12.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:12 smithi001 ceph-mon[112165]: pgmap v1295: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 97 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 23/420 objects degraded (5.476%); 2482/420 objects misplaced (590.952%); 0 B/s, 9 objects/s recovering 2024-09-14T04:24:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:12 smithi001 ceph-mon[107399]: pgmap v1295: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 97 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 23/420 objects degraded (5.476%); 2482/420 objects misplaced (590.952%); 0 B/s, 9 objects/s recovering 2024-09-14T04:24:12.493 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:12.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:12 smithi012 ceph-mon[101180]: pgmap v1295: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 97 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 23/420 objects degraded (5.476%); 2482/420 objects misplaced (590.952%); 0 B/s, 9 objects/s recovering 2024-09-14T04:24:12.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:12.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:12.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:12.770 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:12.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:12.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:12.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:12.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:13.019 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:13.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:13.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:13.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:13.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:13.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:13.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:13.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:13.461 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:13.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:13.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:13.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:13.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:13.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:13.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:13.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:13.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:13.906 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:13.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:14.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:14.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:14.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:14.155 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:14.199 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:24:14.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:14.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:14.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:14.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:24:14.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[107399]: pgmap v1296: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 99 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 268 objects/s recovering 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1831562668' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 23/420 objects degraded (5.476%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[112165]: pgmap v1296: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 99 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 268 objects/s recovering 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1831562668' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 23/420 objects degraded (5.476%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:14.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:14.539 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:14 smithi012 ceph-mon[101180]: pgmap v1296: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 99 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 268 objects/s recovering 2024-09-14T04:24:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:14 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1831562668' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:14 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:24:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 23/420 objects degraded (5.476%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:24:14.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:14.664 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:14.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:14.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:14.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:14.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:14.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:14.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:15.018 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:15.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:15.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:15.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:15.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:15.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:15 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:24:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:15 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:24:15.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:15 smithi012 ceph-mon[101180]: osdmap e377: 8 total, 8 up, 6 in 2024-09-14T04:24:15.392 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:24:15.393 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:24:15.393 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[107399]: osdmap e377: 8 total, 8 up, 6 in 2024-09-14T04:24:15.393 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:24:15.393 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:24:15.393 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:15 smithi001 ceph-mon[112165]: osdmap e377: 8 total, 8 up, 6 in 2024-09-14T04:24:15.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:15.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:15.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:15.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:15.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:15.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:15.777 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:15.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:15.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:15.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:15.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:16.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:16.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:16.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:16.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:16.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:16.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:16.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:16.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:16.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:16.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:16 smithi001 ceph-mon[107399]: pgmap v1298: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 102 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 443 objects/s recovering 2024-09-14T04:24:16.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:16 smithi001 ceph-mon[112165]: pgmap v1298: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 102 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 443 objects/s recovering 2024-09-14T04:24:16.510 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:16 smithi012 ceph-mon[101180]: pgmap v1298: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 10 active+remapped+backfill_wait, 102 active+clean; 7.8 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 3078/420 objects degraded (732.857%); 4669/420 objects misplaced (1111.667%); 0 B/s, 443 objects/s recovering 2024-09-14T04:24:16.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:16.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:16.621 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:24:16.622 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:24:16.640 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:24:16.640 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:24:16.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:16.658 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:24:16.658 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:24:16.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:16.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:16.676 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:24:16.676 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:24:16.693 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:24:16.693 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:24:16.710 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:24:16.710 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:24:16.727 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:24:16.728 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:24:16.745 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:24:16.745 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:24:16.756 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:16] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-09-14T04:24:16.774 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:16.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:16.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:16.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:16.926 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:16.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:17.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:17.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:17.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:17.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:24:17.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:17.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:17.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:17.235 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:17.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:17.302 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[107399]: osdmap e378: 8 total, 8 up, 6 in 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1248941055' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[112165]: osdmap e378: 8 total, 8 up, 6 in 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1248941055' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.365 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:17.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:17.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:17.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:17.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:17 smithi012 ceph-mon[101180]: osdmap e378: 8 total, 8 up, 6 in 2024-09-14T04:24:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:17 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1248941055' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:17.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:17.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:17.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:17.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:17.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:17.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:17.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:18.029 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:18.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:18.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:18.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:18.204 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:24:18.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:18.290 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: 12.2 scrub starts 2024-09-14T04:24:18.290 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: 12.2 scrub ok 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: osdmap e379: 8 total, 8 up, 6 in 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: pgmap v1301: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+remapped+backfill_wait, 103 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 43 KiB/s, 723 objects/s recovering 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1978743542' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:24:18.291 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:18 smithi012 ceph-mon[101180]: osdmap e380: 8 total, 8 up, 6 in 2024-09-14T04:24:18.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:18.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:18.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:18.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:18.461 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: 12.2 scrub starts 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: 12.2 scrub ok 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: osdmap e379: 8 total, 8 up, 6 in 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: pgmap v1301: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+remapped+backfill_wait, 103 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 43 KiB/s, 723 objects/s recovering 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1978743542' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[112165]: osdmap e380: 8 total, 8 up, 6 in 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: 12.2 scrub starts 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: 12.2 scrub ok 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: osdmap e379: 8 total, 8 up, 6 in 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: pgmap v1301: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+remapped+backfill_wait, 103 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 43 KiB/s, 723 objects/s recovering 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1978743542' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:24:18.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:18 smithi001 ceph-mon[107399]: osdmap e380: 8 total, 8 up, 6 in 2024-09-14T04:24:18.531 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:18.555 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:18.657 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:18.680 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:18.782 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:18.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:18.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:18.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:18.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:19.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:19.057 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:19.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:19.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:19.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:19.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:19.307 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:19.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:19.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[107399]: 12.5 continuing backfill to osd.3 from (299'9438,371'16907] MIN to 371'16907 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3077/420 objects degraded (732.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[107399]: osdmap e381: 8 total, 8 up, 6 in 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[112165]: 12.5 continuing backfill to osd.3 from (299'9438,371'16907] MIN to 371'16907 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3077/420 objects degraded (732.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:19 smithi001 ceph-mon[112165]: osdmap e381: 8 total, 8 up, 6 in 2024-09-14T04:24:19.515 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:19 smithi012 ceph-mon[101180]: 12.5 continuing backfill to osd.3 from (299'9438,371'16907] MIN to 371'16907 2024-09-14T04:24:19.516 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3077/420 objects degraded (732.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:19.516 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:19 smithi012 ceph-mon[101180]: osdmap e381: 8 total, 8 up, 6 in 2024-09-14T04:24:19.533 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:19.556 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:19.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:19.683 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:19.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:19.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:19.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:19.932 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:19.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:20.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:20.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:20.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:20.181 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:20.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:24:20.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:20.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:20.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:20.429 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[112165]: pgmap v1304: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 64 KiB/s, 838 objects/s recovering 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[107399]: pgmap v1304: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 64 KiB/s, 838 objects/s recovering 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:20.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:20.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:20.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:20 smithi012 ceph-mon[101180]: pgmap v1304: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 7.8 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 3077/420 objects degraded (732.619%); 4669/420 objects misplaced (1111.667%); 64 KiB/s, 838 objects/s recovering 2024-09-14T04:24:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:20.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:20.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:20.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:20.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:20.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:20.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:20.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:20.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:21.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:21.053 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:21.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:21.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:21.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:21.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:21.302 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:21.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:21.429 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:21.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:21.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:21.656 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:21.680 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:21.782 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:21.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:21.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:21.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:21.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:22.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:22.054 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:22.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:22.177 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:22.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.303 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:22.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:22.404 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:22.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:22.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:22.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:22 smithi012 ceph-mon[101180]: pgmap v1305: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 4669/381 objects misplaced (1225.459%); 49 KiB/s, 636 objects/s recovering 2024-09-14T04:24:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:22.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:22.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.811 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:22.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:22.855 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[112165]: pgmap v1305: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 4669/381 objects misplaced (1225.459%); 49 KiB/s, 636 objects/s recovering 2024-09-14T04:24:22.856 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.856 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[107399]: pgmap v1305: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 4669/381 objects misplaced (1225.459%); 49 KiB/s, 636 objects/s recovering 2024-09-14T04:24:22.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:22.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:22.938 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:22.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:23.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:23.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:23.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:23.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:23.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:23.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:23.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:23.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:23.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:23.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:23.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:23.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:23.695 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:23.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:23.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:23 smithi012 ceph-mon[101180]: osdmap e382: 8 total, 8 up, 6 in 2024-09-14T04:24:23.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:23.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:23.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:23.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:23 smithi001 ceph-mon[107399]: osdmap e382: 8 total, 8 up, 6 in 2024-09-14T04:24:23.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:23 smithi001 ceph-mon[112165]: osdmap e382: 8 total, 8 up, 6 in 2024-09-14T04:24:24.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:24.071 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:24.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:24.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:24.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:24.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:24.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:24.454 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:24.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:24.580 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:24.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:24.704 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:24.806 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:24.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: pgmap v1307: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 577 objects/s recovering 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: 1.11 scrub starts 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: 1.11 scrub ok 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: osdmap e383: 8 total, 8 up, 6 in 2024-09-14T04:24:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:24.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:24.953 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:24.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: pgmap v1307: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 577 objects/s recovering 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: 1.11 scrub starts 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: 1.11 scrub ok 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: osdmap e383: 8 total, 8 up, 6 in 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: pgmap v1307: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 577 objects/s recovering 2024-09-14T04:24:24.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: 1.11 scrub starts 2024-09-14T04:24:24.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: 1.11 scrub ok 2024-09-14T04:24:24.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: 12.1 continuing backfill to osd.1 from (301'10980,370'16794] MIN to 370'16794 2024-09-14T04:24:24.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: osdmap e383: 8 total, 8 up, 6 in 2024-09-14T04:24:24.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:24:25.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:25.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:25.110 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:25.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:25.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:25.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:25.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:25.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:25.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:25.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:25.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:25.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:25.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:25.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:25.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:25.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:25 smithi012 ceph-mon[101180]: osdmap e384: 8 total, 8 up, 6 in 2024-09-14T04:24:25.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:25.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:25.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:25.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:25 smithi001 ceph-mon[107399]: osdmap e384: 8 total, 8 up, 6 in 2024-09-14T04:24:25.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:25 smithi001 ceph-mon[112165]: osdmap e384: 8 total, 8 up, 6 in 2024-09-14T04:24:26.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:26.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:26.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:26.201 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:26.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:26.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:26.327 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:26.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:26.452 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:26.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:26.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:26.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:26.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:26.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:26.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:26.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:26.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:26 smithi012 ceph-mon[101180]: pgmap v1310: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 283 objects/s recovering 2024-09-14T04:24:26.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:26 smithi012 ceph-mon[101180]: osdmap e385: 8 total, 8 up, 6 in 2024-09-14T04:24:26.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:26.956 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:26 smithi001 ceph-mon[107399]: pgmap v1310: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 283 objects/s recovering 2024-09-14T04:24:26.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:26 smithi001 ceph-mon[107399]: osdmap e385: 8 total, 8 up, 6 in 2024-09-14T04:24:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:26] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-09-14T04:24:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:26 smithi001 ceph-mon[112165]: pgmap v1310: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 3077/381 objects degraded (807.612%); 1738/381 objects misplaced (456.168%); 0 B/s, 283 objects/s recovering 2024-09-14T04:24:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:26 smithi001 ceph-mon[112165]: osdmap e385: 8 total, 8 up, 6 in 2024-09-14T04:24:27.059 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.083 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:27.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:27.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:27.208 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:27.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:27.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:27.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:27.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:27.551 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:27.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:27.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:27.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:27.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:27.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:27 smithi012 ceph-mon[101180]: osdmap e386: 8 total, 8 up, 6 in 2024-09-14T04:24:27.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:27.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:27.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:27.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:27.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:27 smithi001 ceph-mon[107399]: osdmap e386: 8 total, 8 up, 6 in 2024-09-14T04:24:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:27 smithi001 ceph-mon[112165]: osdmap e386: 8 total, 8 up, 6 in 2024-09-14T04:24:28.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:28.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:28.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:28.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:28.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:28.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:28.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:28.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:28.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:28.549 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:28.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:28.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:28.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:28.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:28 smithi012 ceph-mon[101180]: 12.15 continuing backfill to osd.3 from (299'9438,371'16952] MIN to 371'16952 2024-09-14T04:24:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:28 smithi012 ceph-mon[101180]: pgmap v1313: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 506 objects/s recovering 2024-09-14T04:24:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:28 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:24:28.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:28.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:28.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[107399]: 12.15 continuing backfill to osd.3 from (299'9438,371'16952] MIN to 371'16952 2024-09-14T04:24:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[107399]: pgmap v1313: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 506 objects/s recovering 2024-09-14T04:24:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:24:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[112165]: 12.15 continuing backfill to osd.3 from (299'9438,371'16952] MIN to 371'16952 2024-09-14T04:24:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[112165]: pgmap v1313: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 506 objects/s recovering 2024-09-14T04:24:28.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:28 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3077/381 objects degraded (807.612%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:24:29.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:29.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:29.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:29.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:29.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:29.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:29.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:29.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:29.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:29.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:29.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:29.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:29.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:29.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:29.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:29.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:29.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:29.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:30.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:30.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:30.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:30.171 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:30.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:30.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:24:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:30.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:30.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:30.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:30.500 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:30.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:30.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:30.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:30.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:30.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:30.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:30 smithi012 ceph-mon[101180]: pgmap v1314: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 341 objects/s recovering 2024-09-14T04:24:30.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:30.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:30.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:30 smithi001 ceph-mon[107399]: pgmap v1314: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 341 objects/s recovering 2024-09-14T04:24:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:30 smithi001 ceph-mon[112165]: pgmap v1314: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 341 objects/s recovering 2024-09-14T04:24:31.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:31.049 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:31.151 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:31.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:31.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:31.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:31.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:31.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:31.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:31.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:31.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:31.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:31.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:31.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:31.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:31.922 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:31.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:32.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:32.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:32.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:32.172 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:32.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:32.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:32.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:32.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:32.545 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:32.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:32.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:32.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:32.794 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:32.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:32 smithi012 ceph-mon[101180]: pgmap v1315: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 292 objects/s recovering 2024-09-14T04:24:32.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:32.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:32.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:32 smithi001 ceph-mon[107399]: pgmap v1315: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 292 objects/s recovering 2024-09-14T04:24:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:32 smithi001 ceph-mon[112165]: pgmap v1315: 119 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 292 objects/s recovering 2024-09-14T04:24:33.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:33.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:33.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:33.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:33.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:33.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:33.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:33.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:33.423 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:33.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:33.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:33.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:33.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:33.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:33.798 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:33.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:33.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:34.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:34.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:34.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:34.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:34.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:34.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:34.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:34.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:34.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:34.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:34.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:34.646 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:34.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:34.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:34.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:34 smithi012 ceph-mon[101180]: pgmap v1316: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 409 objects/s recovering 2024-09-14T04:24:34.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:34.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:34.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:34 smithi001 ceph-mon[107399]: pgmap v1316: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 409 objects/s recovering 2024-09-14T04:24:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:34 smithi001 ceph-mon[112165]: pgmap v1316: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 409 objects/s recovering 2024-09-14T04:24:35.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:35.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:35.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:35.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:35.167 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:35.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:35.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:35.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:35.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:35.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:35.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:35.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:35.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:35.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:35.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:35.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:35.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:35.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:35.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:36.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:36.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:36.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:36.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[107399]: pgmap v1317: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 330 objects/s recovering 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:35 smithi001 ceph-mon[112165]: pgmap v1317: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 330 objects/s recovering 2024-09-14T04:24:36.267 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:36.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:36.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:35 smithi012 ceph-mon[101180]: pgmap v1317: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized+remapped+backfill_wait, 11 active+remapped+backfill_wait, 104 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 1738/381 objects misplaced (456.168%); 0 B/s, 330 objects/s recovering 2024-09-14T04:24:36.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:36.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:36.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:36.539 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:36.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:36.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:36.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:36.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:36.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:36.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:36.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:36.953 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:36] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-14T04:24:37.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:37.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:37.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:37.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:37.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:36 smithi001 ceph-mon[107399]: osdmap e387: 8 total, 8 up, 6 in 2024-09-14T04:24:37.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:36 smithi001 ceph-mon[112165]: osdmap e387: 8 total, 8 up, 6 in 2024-09-14T04:24:37.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:37.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:36 smithi012 ceph-mon[101180]: osdmap e387: 8 total, 8 up, 6 in 2024-09-14T04:24:37.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:37.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:37.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:37.539 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:37.641 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:37.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:37.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:37.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:37.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:37.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:37.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:37.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:38.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:38.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:38.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:38.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:38.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[112165]: osdmap e388: 8 total, 8 up, 6 in 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[112165]: 12.15 scrub starts 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[112165]: 12.15 scrub ok 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[112165]: pgmap v1320: 119 pgs: 2 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 223 objects/s recovering 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[107399]: osdmap e388: 8 total, 8 up, 6 in 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[107399]: 12.15 scrub starts 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[107399]: 12.15 scrub ok 2024-09-14T04:24:38.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:37 smithi001 ceph-mon[107399]: pgmap v1320: 119 pgs: 2 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 223 objects/s recovering 2024-09-14T04:24:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:37 smithi012 ceph-mon[101180]: osdmap e388: 8 total, 8 up, 6 in 2024-09-14T04:24:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:37 smithi012 ceph-mon[101180]: 12.15 scrub starts 2024-09-14T04:24:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:37 smithi012 ceph-mon[101180]: 12.15 scrub ok 2024-09-14T04:24:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:37 smithi012 ceph-mon[101180]: pgmap v1320: 119 pgs: 2 active+recovering+undersized+remapped, 11 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 223 objects/s recovering 2024-09-14T04:24:38.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:38.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:38.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:38.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:38.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:38.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:38.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:38.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:38.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:38.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:38.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:38.888 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:38.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:38.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:38.969 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:38 smithi001 ceph-mon[107399]: 12.5 scrub starts 2024-09-14T04:24:38.969 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:38 smithi001 ceph-mon[107399]: 12.5 scrub ok 2024-09-14T04:24:39.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:39.036 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:39.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:39.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:39.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:38 smithi001 ceph-mon[112165]: 12.5 scrub starts 2024-09-14T04:24:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:38 smithi001 ceph-mon[112165]: 12.5 scrub ok 2024-09-14T04:24:39.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:39.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:38 smithi012 ceph-mon[101180]: 12.5 scrub starts 2024-09-14T04:24:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:38 smithi012 ceph-mon[101180]: 12.5 scrub ok 2024-09-14T04:24:39.387 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:39.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:39.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:39.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:39.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:39.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:39.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:39.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:39.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:39.913 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:39.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:40.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:40.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:40.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:40.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:40.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:40.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:39 smithi012 ceph-mon[101180]: 12.6 scrub starts 2024-09-14T04:24:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:39 smithi012 ceph-mon[101180]: 12.6 scrub ok 2024-09-14T04:24:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:39 smithi012 ceph-mon[101180]: osdmap e389: 8 total, 8 up, 6 in 2024-09-14T04:24:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:39 smithi012 ceph-mon[101180]: pgmap v1322: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 8 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 89 objects/s recovering 2024-09-14T04:24:40.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:40.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[112165]: 12.6 scrub starts 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[112165]: 12.6 scrub ok 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[112165]: osdmap e389: 8 total, 8 up, 6 in 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[112165]: pgmap v1322: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 8 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 89 objects/s recovering 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[107399]: 12.6 scrub starts 2024-09-14T04:24:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[107399]: 12.6 scrub ok 2024-09-14T04:24:40.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[107399]: osdmap e389: 8 total, 8 up, 6 in 2024-09-14T04:24:40.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:39 smithi001 ceph-mon[107399]: pgmap v1322: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 8 active+remapped+backfill_wait, 106 active+clean; 6.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 158/381 objects misplaced (41.470%); 0 B/s, 89 objects/s recovering 2024-09-14T04:24:40.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:40.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:40.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:40.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:40.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:40.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:40.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:40.913 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:40.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:41.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:41.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:41.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[112165]: osdmap e390: 8 total, 8 up, 6 in 2024-09-14T04:24:41.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[112165]: 12.10 scrub starts 2024-09-14T04:24:41.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[112165]: 12.10 scrub ok 2024-09-14T04:24:41.044 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[107399]: osdmap e390: 8 total, 8 up, 6 in 2024-09-14T04:24:41.044 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[107399]: 12.10 scrub starts 2024-09-14T04:24:41.044 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:40 smithi001 ceph-mon[107399]: 12.10 scrub ok 2024-09-14T04:24:41.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:41.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:41.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:41.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:40 smithi012 ceph-mon[101180]: osdmap e390: 8 total, 8 up, 6 in 2024-09-14T04:24:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:40 smithi012 ceph-mon[101180]: 12.10 scrub starts 2024-09-14T04:24:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:40 smithi012 ceph-mon[101180]: 12.10 scrub ok 2024-09-14T04:24:41.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:41.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:41.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:41.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:41.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:41.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:41.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:41.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:41.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:41.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:41.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:42.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:42.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:42.055 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[112165]: 12.16 scrub starts 2024-09-14T04:24:42.055 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[112165]: 12.16 scrub ok 2024-09-14T04:24:42.055 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[112165]: 12.0 scrub starts 2024-09-14T04:24:42.055 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[112165]: 12.0 scrub ok 2024-09-14T04:24:42.055 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[112165]: pgmap v1324: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 4.2 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70/249 objects misplaced (28.112%) 2024-09-14T04:24:42.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[107399]: 12.16 scrub starts 2024-09-14T04:24:42.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[107399]: 12.16 scrub ok 2024-09-14T04:24:42.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[107399]: 12.0 scrub starts 2024-09-14T04:24:42.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[107399]: 12.0 scrub ok 2024-09-14T04:24:42.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:41 smithi001 ceph-mon[107399]: pgmap v1324: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 4.2 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70/249 objects misplaced (28.112%) 2024-09-14T04:24:42.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:42.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:42.270 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:42.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:41 smithi012 ceph-mon[101180]: 12.16 scrub starts 2024-09-14T04:24:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:41 smithi012 ceph-mon[101180]: 12.16 scrub ok 2024-09-14T04:24:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:41 smithi012 ceph-mon[101180]: 12.0 scrub starts 2024-09-14T04:24:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:41 smithi012 ceph-mon[101180]: 12.0 scrub ok 2024-09-14T04:24:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:41 smithi012 ceph-mon[101180]: pgmap v1324: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 4.2 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70/249 objects misplaced (28.112%) 2024-09-14T04:24:42.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:42.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:42.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:42.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:42.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:42.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:42.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:42.799 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:42.800 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:42.805 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:42.832 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:24:42.832 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:24:42.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:42.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:42.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:43.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:43.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:43.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:43.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:43.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:43.315 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:43.417 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:43.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:43.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:43.568 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:43.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:43.693 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:43.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:43.822 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:43.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:43.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:43.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:44.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:44.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:44.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:44.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:44.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:44.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:44.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.461 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:44.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:44.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:44.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:44.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:44.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:44.714 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:44.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:44.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:44.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:44 smithi012 ceph-mon[101180]: pgmap v1325: 119 pgs: 3 peering, 116 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:44.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:44.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:44.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:44.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:44 smithi001 ceph-mon[112165]: pgmap v1325: 119 pgs: 3 peering, 116 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:44.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:44 smithi001 ceph-mon[107399]: pgmap v1325: 119 pgs: 3 peering, 116 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:45.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:24:45.071 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:45.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:45.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:45.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:45.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:45.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:45.445 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:45.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:45.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:45.572 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:24:45.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:45 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:24:45.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:24:45.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:45 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:24:45.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:45.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:45.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:45.829 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:24:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:45 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:24:45.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:45.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:45.980 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:45.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:46.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:46.108 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:46.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:46.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:46.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:46.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:46.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:46.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:46.488 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:46.535 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_7' removed 2024-09-14T04:24:46.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.655 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:46.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:46.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:46.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: pgmap v1326: 119 pgs: 119 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: osdmap e391: 8 total, 8 up, 6 in 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:24:46.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:24:46.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:46.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:46.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:46] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: pgmap v1326: 119 pgs: 119 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: osdmap e391: 8 total, 8 up, 6 in 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:24:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: pgmap v1326: 119 pgs: 119 active+clean; 634 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: osdmap e391: 8 total, 8 up, 6 in 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:24:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:24:47.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:47.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:47.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:47.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:47.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:47.387 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:47.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:47.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:47.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:47.558 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:24:47.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:47.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:47.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.794 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:47.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:24:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:47 smithi012 ceph-mon[101180]: osdmap e392: 8 total, 8 up, 6 in 2024-09-14T04:24:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1454346375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:47.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:47 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:47.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:47.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:47.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[107399]: osdmap e392: 8 total, 8 up, 6 in 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1454346375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3165840380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[112165]: osdmap e392: 8 total, 8 up, 6 in 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1454346375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:47 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:24:48.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:48.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:48.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:48.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:48.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:48.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:48.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:48.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:48.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:48.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:48.566 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:48.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:48.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: pgmap v1329: 93 pgs: 3 remapped+peering, 2 unknown, 3 peering, 85 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: osdmap e393: 8 total, 8 up, 6 in 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: 1.1 scrub starts 2024-09-14T04:24:48.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[112165]: 1.1 scrub ok 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: pgmap v1329: 93 pgs: 3 remapped+peering, 2 unknown, 3 peering, 85 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: osdmap e393: 8 total, 8 up, 6 in 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: 1.1 scrub starts 2024-09-14T04:24:48.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:48 smithi001 ceph-mon[107399]: 1.1 scrub ok 2024-09-14T04:24:48.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:48.823 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: pgmap v1329: 93 pgs: 3 remapped+peering, 2 unknown, 3 peering, 85 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: osdmap e393: 8 total, 8 up, 6 in 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: 1.1 scrub starts 2024-09-14T04:24:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:48 smithi012 ceph-mon[101180]: 1.1 scrub ok 2024-09-14T04:24:48.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:48.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:48.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:49.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:49.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:49.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:49.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:49.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:49.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:49.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:49.464 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:49.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:49.566 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:49.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:49.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:49.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:49.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:49.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:49.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:49 smithi012 ceph-mon[101180]: osdmap e394: 8 total, 8 up, 6 in 2024-09-14T04:24:49.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:49.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:49.883 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:49 smithi001 ceph-mon[112165]: osdmap e394: 8 total, 8 up, 6 in 2024-09-14T04:24:49.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:49 smithi001 ceph-mon[107399]: osdmap e394: 8 total, 8 up, 6 in 2024-09-14T04:24:49.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:49.974 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:49.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:50.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:50.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:50.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:50.226 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:50.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:50.351 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:50.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:50.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:50.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:50.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:50.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:50.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:50.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: pgmap v1332: 93 pgs: 3 remapped+peering, 2 unknown, 4 peering, 84 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: osdmap e395: 8 total, 8 up, 6 in 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: 1.11 scrub starts 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: 1.11 scrub ok 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: pgmap v1332: 93 pgs: 3 remapped+peering, 2 unknown, 4 peering, 84 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: osdmap e395: 8 total, 8 up, 6 in 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: 1.11 scrub starts 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: 1.11 scrub ok 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:50.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:50.737 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:50.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:24:50.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: pgmap v1332: 93 pgs: 3 remapped+peering, 2 unknown, 4 peering, 84 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: osdmap e395: 8 total, 8 up, 6 in 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: 1.11 scrub starts 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: 1.11 scrub ok 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:50.895 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:50.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:50.997 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:51.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:51.087 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:51.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:51.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:51.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:51.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:51.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:51.376 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:51.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:51.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:51.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:51.577 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:24:51.577 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:24:51.577 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:24:51.578 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:24:51.592 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:24:51.593 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:24:51.608 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:24:51.609 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:24:51.626 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:24:51.626 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:24:51.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:51.643 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:24:51.643 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:24:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:51.660 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:24:51.660 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:24:51.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:24:51.677 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:24:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:24:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[107399]: 1.e scrub ok 2024-09-14T04:24:51.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1135907137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:51.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:24:51.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[112165]: 1.e scrub ok 2024-09-14T04:24:51.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1135907137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:51.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:51.782 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:51 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:24:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:51 smithi012 ceph-mon[101180]: 1.e scrub ok 2024-09-14T04:24:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1135907137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:24:51.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:51.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:51.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:52.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:52.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:52.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:52.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:52.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:52.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:52.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:52.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:52.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:52.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:52.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:52.571 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:24:52.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:52.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:52.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:52.802 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:52.852 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:52 smithi012 ceph-mon[101180]: pgmap v1334: 93 pgs: 3 remapped+peering, 1 peering, 89 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 7.7 KiB/s rd, 22 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-09-14T04:24:52.853 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2218661622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.853 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:52.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[112165]: pgmap v1334: 93 pgs: 3 remapped+peering, 1 peering, 89 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 7.7 KiB/s rd, 22 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-09-14T04:24:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2218661622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[107399]: pgmap v1334: 93 pgs: 3 remapped+peering, 1 peering, 89 active+clean; 608 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 7.7 KiB/s rd, 22 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-09-14T04:24:52.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2218661622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:24:52.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:52.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:53.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.113 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:53.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:53.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:53.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:53.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:53.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:53.395 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:24:53.396 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:24:53.396 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2024-09-14T04:24:53.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_8 16 2024-09-14T04:24:53.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:53.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:53.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:53.629 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:24:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:24:53.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[107399]: osdmap e396: 8 total, 8 up, 6 in 2024-09-14T04:24:53.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:24:53.725 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:24:53.725 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:53 smithi001 ceph-mon[112165]: osdmap e396: 8 total, 8 up, 6 in 2024-09-14T04:24:53.730 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:53.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:53.759 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:53.763 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:24:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:53 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:24:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:24:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:53 smithi012 ceph-mon[101180]: osdmap e396: 8 total, 8 up, 6 in 2024-09-14T04:24:53.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:53.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:53.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:53.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:54.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:24:54.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:24:54.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:54.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:54.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:54.396 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:54.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:54.524 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:54.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.654 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[107399]: pgmap v1336: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 41 KiB/s, 0 objects/s recovering 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[112165]: pgmap v1336: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 41 KiB/s, 0 objects/s recovering 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:24:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:54 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:24:54.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:54.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:54 smithi012 ceph-mon[101180]: pgmap v1336: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 41 KiB/s, 0 objects/s recovering 2024-09-14T04:24:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:54 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:24:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:54 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:24:54.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:54.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:54.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:55.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:55.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:55.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:55.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:55.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:55.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:55.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:55.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:55.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:55.537 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:55.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:55.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:55.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:55.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:55.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:55.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:55.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:55.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:56.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:56.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:56.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:56.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:56.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:56.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:56.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:56.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:56.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:56.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:56.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:56.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:56.677 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:56.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:56.806 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:56 smithi012 ceph-mon[101180]: pgmap v1337: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:24:56.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:56.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:56.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:56.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:56 smithi001 ceph-mon[107399]: pgmap v1337: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:24:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:56 smithi001 ceph-mon[112165]: pgmap v1337: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:24:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:24:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:24:56] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.51.0" 2024-09-14T04:24:57.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:57.062 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:57.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:57.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.193 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:57.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:57.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:57.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:57.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:57.448 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:57.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:57.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:57.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:57.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:57.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:57.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:57.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:57.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:57.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:58.063 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:58.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:58.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:58.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:58.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:58.345 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:58.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:24:58.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:24:58.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:58.588 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:24:58.600 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:58.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:24:58.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:24:58.728 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:58.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:24:58.829 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:58 smithi012 ceph-mon[101180]: pgmap v1338: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:24:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3210802841' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:58 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:58 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:58 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:58.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:58.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:58.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[107399]: pgmap v1338: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:24:58.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3210802841' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[112165]: pgmap v1338: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:24:58.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3210802841' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:24:58.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:58 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:58.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:58.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:59.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.111 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:59.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:59.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:59.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:24:59.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:24:59.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:24:59.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:24:59.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:24:59.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:24:59.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:24:59.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:24:59.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:24:59.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:24:59.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[112165]: osdmap e397: 8 total, 8 up, 6 in 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[107399]: osdmap e397: 8 total, 8 up, 6 in 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:24:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:59.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:24:59.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:59 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:24:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:59 smithi012 ceph-mon[101180]: osdmap e397: 8 total, 8 up, 6 in 2024-09-14T04:24:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:59.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:24:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:24:59.864 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:24:59.966 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:24:59.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:24:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:24:59.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:00.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:00.117 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:00.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:00.243 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:00.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:00.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:00.369 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:00.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:00.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:00.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:00.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:00 smithi001 ceph-mon[107399]: pgmap v1340: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:25:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-14T04:25:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:00 smithi001 ceph-mon[112165]: pgmap v1340: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:25:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-14T04:25:00.722 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:00.744 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:00.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:00.873 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:00.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:00.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:00.929 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:00.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:00.978 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_8' created 2024-09-14T04:25:01.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:00 smithi012 ceph-mon[101180]: pgmap v1340: 93 pgs: 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-14T04:25:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-14T04:25:01.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:01.134 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:01.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:01.261 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:01.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:01.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:01.389 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:01.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:01.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:01.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:01.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:01.747 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:01.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:01.861 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:25:01.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:01.926 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:01.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:02.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:02.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:02.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:02.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[107399]: osdmap e398: 8 total, 8 up, 6 in 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[107399]: pgmap v1342: 109 pgs: 6 creating+peering, 10 unknown, 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[112165]: osdmap e398: 8 total, 8 up, 6 in 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[112165]: pgmap v1342: 109 pgs: 6 creating+peering, 10 unknown, 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:02.141 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:01 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:25:02.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:02.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:02.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:02.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.306 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:02.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1343396685' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-14T04:25:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:01 smithi012 ceph-mon[101180]: osdmap e398: 8 total, 8 up, 6 in 2024-09-14T04:25:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:01 smithi012 ceph-mon[101180]: pgmap v1342: 109 pgs: 6 creating+peering, 10 unknown, 93 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:02.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:01 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:25:02.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:02.433 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:02.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:02.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:02.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:02.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:02.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2024-09-14T04:25:02.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:02.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:02.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:02.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:02.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:02.966 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:25:02.966 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:25:02.967 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:25:02.967 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:25:02.967 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:25:03.041 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:03.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:03.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:03.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:03.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:03.317 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:03.321 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:02 smithi001 ceph-mon[107399]: osdmap e399: 8 total, 8 up, 6 in 2024-09-14T04:25:03.322 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:02 smithi001 ceph-mon[112165]: osdmap e399: 8 total, 8 up, 6 in 2024-09-14T04:25:03.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:02 smithi012 ceph-mon[101180]: osdmap e399: 8 total, 8 up, 6 in 2024-09-14T04:25:03.419 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:03.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:03.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:03.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:03.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:03.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:03.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:25:03.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:03.797 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:03.797 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:03.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:03.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:03.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:03.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:04.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:04.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:04.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:04.192 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:03 smithi001 ceph-mon[107399]: osdmap e400: 8 total, 8 up, 6 in 2024-09-14T04:25:04.192 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:03 smithi001 ceph-mon[107399]: pgmap v1345: 109 pgs: 2 creating+activating, 6 creating+peering, 5 unknown, 96 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:04.192 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:03 smithi001 ceph-mon[112165]: osdmap e400: 8 total, 8 up, 6 in 2024-09-14T04:25:04.192 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:03 smithi001 ceph-mon[112165]: pgmap v1345: 109 pgs: 2 creating+activating, 6 creating+peering, 5 unknown, 96 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:04.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:04.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:04.270 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:04.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:04.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:03 smithi012 ceph-mon[101180]: osdmap e400: 8 total, 8 up, 6 in 2024-09-14T04:25:04.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:03 smithi012 ceph-mon[101180]: pgmap v1345: 109 pgs: 2 creating+activating, 6 creating+peering, 5 unknown, 96 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:04.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:04.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:04.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:04.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:04.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:04.704 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:04.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:04.833 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:04.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:04.961 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:04.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:04.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:05.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:05.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:05.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:05.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:05.316 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:05.342 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:05.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:05.471 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:05.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:05.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:05.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:05.725 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:05.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:05.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:05.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:05.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:05.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:05.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:06.053 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:06.053 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:06.053 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[107399]: pgmap v1346: 109 pgs: 2 creating+activating, 6 creating+peering, 101 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:06.054 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:06.054 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:06.054 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:05 smithi001 ceph-mon[112165]: pgmap v1346: 109 pgs: 2 creating+activating, 6 creating+peering, 101 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:06.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:06.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:06.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:06.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:06.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:06.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:05 smithi012 ceph-mon[101180]: pgmap v1346: 109 pgs: 2 creating+activating, 6 creating+peering, 101 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:06.357 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:06.459 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:06.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:06.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:06.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:06.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:06.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:06.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:06.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:06.948 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_8' 2024-09-14T04:25:06.950 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:06] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.51.0" 2024-09-14T04:25:06.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:06.961 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:25:06.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:06.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:06.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:07.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:07.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:07.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:07.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/221448464' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:25:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/221448464' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:25:07.242 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:07.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/221448464' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:06 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:25:07.347 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:25:07.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:07.472 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:07.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:07.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:07.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:07.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:07.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:07.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:07.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:07.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:07.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:08.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:08.107 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:08.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[112165]: osdmap e401: 8 total, 8 up, 6 in 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[112165]: pgmap v1348: 109 pgs: 2 peering, 107 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[112165]: osdmap e402: 8 total, 8 up, 6 in 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:25:08.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[107399]: osdmap e401: 8 total, 8 up, 6 in 2024-09-14T04:25:08.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[107399]: pgmap v1348: 109 pgs: 2 peering, 107 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:08.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:07 smithi001 ceph-mon[107399]: osdmap e402: 8 total, 8 up, 6 in 2024-09-14T04:25:08.233 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:08.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:07 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:25:08.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1661460852' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:25:08.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:07 smithi012 ceph-mon[101180]: osdmap e401: 8 total, 8 up, 6 in 2024-09-14T04:25:08.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:07 smithi012 ceph-mon[101180]: pgmap v1348: 109 pgs: 2 peering, 107 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:08.248 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:07 smithi012 ceph-mon[101180]: osdmap e402: 8 total, 8 up, 6 in 2024-09-14T04:25:08.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:08.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:08.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.382 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:08.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:08.482 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:08.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:08.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:08.634 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:08.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:08.760 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:08.862 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:08.885 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:08.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:08.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:09.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:09.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:09.123 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:25:09.134 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[107399]: osdmap e403: 8 total, 8 up, 6 in 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:25:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:08 smithi001 ceph-mon[112165]: osdmap e403: 8 total, 8 up, 6 in 2024-09-14T04:25:09.237 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:09.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.261 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:09.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:08 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:25:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:08 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:25:09.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:08 smithi012 ceph-mon[101180]: osdmap e403: 8 total, 8 up, 6 in 2024-09-14T04:25:09.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:09.387 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:09.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:09.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:09.512 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:09.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:09.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:09.711 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2024-09-14T04:25:09.713 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:25:09.714 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:25:09.739 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:09.762 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:09.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:09.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:25:09.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:25:09.780 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_306460 2024-09-14T04:25:09.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:25:09.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:25:09.864 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:09.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:09.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:10.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:10.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:10.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:10.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:10.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:10.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:10 smithi012 ceph-mon[101180]: osdmap e404: 8 total, 8 up, 6 in 2024-09-14T04:25:10.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:10 smithi012 ceph-mon[101180]: pgmap v1352: 109 pgs: 3 peering, 106 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:10.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:10.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:10 smithi001 ceph-mon[107399]: osdmap e404: 8 total, 8 up, 6 in 2024-09-14T04:25:10.427 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:10 smithi001 ceph-mon[107399]: pgmap v1352: 109 pgs: 3 peering, 106 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:10.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:10 smithi001 ceph-mon[112165]: osdmap e404: 8 total, 8 up, 6 in 2024-09-14T04:25:10.427 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:10 smithi001 ceph-mon[112165]: pgmap v1352: 109 pgs: 3 peering, 106 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2024-09-14T04:25:10.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:10.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:10.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:10.611 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:10.688 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:10.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:10.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:10.760 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:25:10.760 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:25:10.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3595 3579 223.606 223.688 0.00259395 0.00445472 2024-09-14T04:25:10.798 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:25:10.799 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:25:10.837 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:25:10.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:25:10.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:10.878 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:25:10.878 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:25:10.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:10.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:10.913 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:25:10.914 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:25:10.947 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:25:10.948 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:25:10.983 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:25:10.983 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:25:10.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:11.019 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:25:11.019 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:25:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:11.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:11.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:11.183 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:11.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:11.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:11.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:11.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:11.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:11 smithi001 ceph-mon[112165]: osdmap e405: 8 total, 8 up, 6 in 2024-09-14T04:25:11.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:11 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4059180882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:11 smithi001 ceph-mon[107399]: osdmap e405: 8 total, 8 up, 6 in 2024-09-14T04:25:11.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:11 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4059180882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:11.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:11 smithi012 ceph-mon[101180]: osdmap e405: 8 total, 8 up, 6 in 2024-09-14T04:25:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:11 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4059180882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:11.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:11.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:11.728 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:11.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 6481 6465 201.985 180.375 0.00251573 0.00493717 2024-09-14T04:25:11.830 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:11.880 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:11.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:11.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:12.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:12.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:12.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:12.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:12.212 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:25:12.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:12.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:12.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:12.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:12 smithi001 ceph-mon[107399]: pgmap v1354: 109 pgs: 4 peering, 105 active+clean; 119 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 30 MiB/s wr, 480 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:25:12.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:25:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:12 smithi001 ceph-mon[112165]: pgmap v1354: 109 pgs: 4 peering, 105 active+clean; 119 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 30 MiB/s wr, 480 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:25:12.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:25:12.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:12.565 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:12 smithi012 ceph-mon[101180]: pgmap v1354: 109 pgs: 4 peering, 105 active+clean; 119 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 30 MiB/s wr, 480 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:25:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:25:12.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:12.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:12.781 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 8997 8981 187.07 157.25 0.00164703 0.00530373 2024-09-14T04:25:12.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:12.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:12.935 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:12.974 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:12.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:13.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:13.107 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:13.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:13.242 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:13.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:13.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[107399]: osdmap e406: 8 total, 8 up, 6 in 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:25:13.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:13 smithi001 ceph-mon[112165]: osdmap e406: 8 total, 8 up, 6 in 2024-09-14T04:25:13.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:13.519 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:13 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:13 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2736457078' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:25:13.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:13 smithi012 ceph-mon[101180]: osdmap e406: 8 total, 8 up, 6 in 2024-09-14T04:25:13.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:13.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:13.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:13.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 11530 11514 179.878 158.312 0.00390254 0.00555052 2024-09-14T04:25:13.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:13.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:13.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:13.934 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:13.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:14.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:14.067 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:14.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:14.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:14.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:14.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:14.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:14.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:14.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:14 smithi001 ceph-mon[107399]: pgmap v1356: 109 pgs: 2 peering, 107 active+clean; 259 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 8.4 KiB/s rd, 57 MiB/s wr, 919 op/s; 88 KiB/s, 0 objects/s recovering 2024-09-14T04:25:14.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:14 smithi001 ceph-mon[112165]: pgmap v1356: 109 pgs: 2 peering, 107 active+clean; 259 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 8.4 KiB/s rd, 57 MiB/s wr, 919 op/s; 88 KiB/s, 0 objects/s recovering 2024-09-14T04:25:14.478 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:14.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:14.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:14 smithi012 ceph-mon[101180]: pgmap v1356: 109 pgs: 2 peering, 107 active+clean; 259 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 8.4 KiB/s rd, 57 MiB/s wr, 919 op/s; 88 KiB/s, 0 objects/s recovering 2024-09-14T04:25:14.619 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:14.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:14.762 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:14.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 14439 14423 180.262 181.812 0.00590477 0.00554155 2024-09-14T04:25:14.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:14.896 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:14.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:14.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:15.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:15.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:15.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:15.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:15.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:15.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:15.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:15.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:15.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:15.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:15.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:15.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 17297 17281 179.986 178.625 0.00322287 0.00554986 2024-09-14T04:25:15.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.843 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:15.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:15.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:15.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:15.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:16.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:16.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:16.150 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:16.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:16.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:16.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:16.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:16.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:16.529 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:16.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:16.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:16 smithi001 ceph-mon[112165]: pgmap v1357: 109 pgs: 109 active+clean; 533 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6.0 KiB/s rd, 85 MiB/s wr, 1.36k op/s; 63 KiB/s, 0 objects/s recovering 2024-09-14T04:25:16.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:16 smithi001 ceph-mon[107399]: pgmap v1357: 109 pgs: 109 active+clean; 533 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6.0 KiB/s rd, 85 MiB/s wr, 1.36k op/s; 63 KiB/s, 0 objects/s recovering 2024-09-14T04:25:16.764 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:25:16.764 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:25:16.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:16.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 19880 19864 177.335 161.438 0.00566532 0.00563295 2024-09-14T04:25:16.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:16 smithi012 ceph-mon[101180]: pgmap v1357: 109 pgs: 109 active+clean; 533 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6.0 KiB/s rd, 85 MiB/s wr, 1.36k op/s; 63 KiB/s, 0 objects/s recovering 2024-09-14T04:25:16.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:16.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:16.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:16] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.51.0" 2024-09-14T04:25:17.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:17.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:17.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:17.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:17.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:17.342 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:17.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:17.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.486 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:17.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:17.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:25:17.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:17.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:17.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:17.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:17.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:17.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 15 22194 22179 173.253 144.688 0.00939895 0.00576573 2024-09-14T04:25:17.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:17.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:17.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:17.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:18.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:18.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:18.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:18.182 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:25:18.189 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:18.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:18.291 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:18.322 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:18.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:18.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:18.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:18.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:18.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:18.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:18.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 24363 24347 169.057 135.5 0.0101743 0.0059088 2024-09-14T04:25:18.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:18 smithi012 ceph-mon[101180]: pgmap v1358: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 139 MiB/s wr, 2.23k op/s; 50 KiB/s, 0 objects/s recovering 2024-09-14T04:25:18.866 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:18.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:18.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:18 smithi001 ceph-mon[107399]: pgmap v1358: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 139 MiB/s wr, 2.23k op/s; 50 KiB/s, 0 objects/s recovering 2024-09-14T04:25:18.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:18 smithi001 ceph-mon[112165]: pgmap v1358: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 139 MiB/s wr, 2.23k op/s; 50 KiB/s, 0 objects/s recovering 2024-09-14T04:25:19.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:19.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:19.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:19.141 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:19.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:19.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:19.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:19.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.393 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:19.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:19.494 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:19.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:19.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:19.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:19.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:19.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 25793 25777 161.088 89.375 0.0537326 0.00617893 2024-09-14T04:25:19.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:19.894 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:19.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:19.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:20.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:20.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:20.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:20.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:20.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:20.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:20.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:20.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:20.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:20.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:20.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:20.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:20.746 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:20.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 27874 27858 158.266 130.062 0.00871681 0.00631235 2024-09-14T04:25:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:20 smithi012 ceph-mon[101180]: pgmap v1359: 109 pgs: 109 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 34 KiB/s, 0 objects/s recovering 2024-09-14T04:25:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:20.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:20.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:20.871 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:20.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[112165]: pgmap v1359: 109 pgs: 109 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 34 KiB/s, 0 objects/s recovering 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[107399]: pgmap v1359: 109 pgs: 109 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 34 KiB/s, 0 objects/s recovering 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:25:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:20.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:21.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:21.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:21.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:21.235 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:21.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:21.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:21.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:21.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:21.550 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:21.652 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:21.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:21.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:21.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 29632 29616 154.233 109.875 0.00444165 0.00647839 2024-09-14T04:25:21.811 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:21.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:21.954 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:21.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:21.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:22.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:22.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:22.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:25:22.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:22.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:22.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:22.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:22.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:22.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:22.542 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:25:22.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:22.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:22.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:22 smithi001 ceph-mon[107399]: pgmap v1360: 109 pgs: 109 active+clean; 1.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:25:22.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:22 smithi001 ceph-mon[112165]: pgmap v1360: 109 pgs: 109 active+clean; 1.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:25:22.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:22.782 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 31798 31782 152.781 135.375 0.00957613 0.00653837 2024-09-14T04:25:22.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:22.801 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:22 smithi012 ceph-mon[101180]: pgmap v1360: 109 pgs: 109 active+clean; 1.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:25:22.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:22.936 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:22.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:23.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:23.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:23.173 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:23.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:23.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:23.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:23.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:23.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:23.597 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:23.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:23.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:23.763 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:23.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 33767 33751 150.65 123.062 0.0104493 0.00663202 2024-09-14T04:25:23.865 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:23.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:23.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:24.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:24.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:24.109 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:24.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:24.192 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:24.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:24.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:24.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:24.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:24.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:24.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:24.555 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:25:24.599 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:24.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:24.724 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:24.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 35456 35440 147.643 105.562 0.0020269 0.00671271 2024-09-14T04:25:24.827 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:24 smithi012 ceph-mon[101180]: pgmap v1361: 109 pgs: 109 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-14T04:25:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:25:24.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:24.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:24.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:24 smithi001 ceph-mon[112165]: pgmap v1361: 109 pgs: 109 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-14T04:25:24.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:25:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:24 smithi001 ceph-mon[107399]: pgmap v1361: 109 pgs: 109 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-09-14T04:25:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:25:24.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:24.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:24.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:25.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.101 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:25.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:25.202 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:25.224 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:25.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:25.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:25.451 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:25:25.451 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:25:25.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:25.474 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:25.576 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:25.602 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:25.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:25.727 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:25.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 35491 35475 138.553 2.1875 0.00244501 0.00671571 2024-09-14T04:25:25.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:25.841 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:25:25.842 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:25 smithi012 ceph-mon[101180]: osdmap e407: 8 total, 8 up, 6 in 2024-09-14T04:25:25.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:25.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:25:25.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:25 smithi001 ceph-mon[107399]: osdmap e407: 8 total, 8 up, 6 in 2024-09-14T04:25:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2885331581' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:25:25.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:25 smithi001 ceph-mon[112165]: osdmap e407: 8 total, 8 up, 6 in 2024-09-14T04:25:25.977 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:25.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:26.079 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:26.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:26.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:26.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:25:26.208 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:26.236 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:26.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:26.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:26.374 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:25:26.463 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:26.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:26.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:26.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:26.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:26.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:26 smithi001 ceph-mon[107399]: pgmap v1363: 109 pgs: 1 peering, 108 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-14T04:25:26.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:26 smithi001 ceph-mon[107399]: osdmap e408: 8 total, 8 up, 6 in 2024-09-14T04:25:26.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:26 smithi001 ceph-mon[112165]: pgmap v1363: 109 pgs: 1 peering, 108 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-14T04:25:26.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:26 smithi001 ceph-mon[112165]: osdmap e408: 8 total, 8 up, 6 in 2024-09-14T04:25:26.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:26.761 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:26.777 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:26 smithi012 ceph-mon[101180]: pgmap v1363: 109 pgs: 1 peering, 108 active+clean; 1.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-14T04:25:26.778 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:26 smithi012 ceph-mon[101180]: osdmap e408: 8 total, 8 up, 6 in 2024-09-14T04:25:26.783 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 15 35724 35709 131.263 14.625 0.00679218 0.0076114 2024-09-14T04:25:26.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:26.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:26.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:26] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.51.0" 2024-09-14T04:25:27.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:27.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:27.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:27.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:27.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:27.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:27.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:27.453 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:27.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:25:27.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:27.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:27.692 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:27 smithi001 ceph-mon[107399]: osdmap e409: 8 total, 8 up, 6 in 2024-09-14T04:25:27.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:27 smithi001 ceph-mon[112165]: osdmap e409: 8 total, 8 up, 6 in 2024-09-14T04:25:27.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:27.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:27.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:27.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 37614 37598 130.529 118.062 0.00715062 0.00765552 2024-09-14T04:25:27.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:27 smithi012 ceph-mon[101180]: osdmap e409: 8 total, 8 up, 6 in 2024-09-14T04:25:27.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:27.911 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:27.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:27.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:28.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:28.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:28.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:28.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:28.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:28.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:28.323 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:28.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:28.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:28.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:28.587 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:25:28.587 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:25:28.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:28.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:28.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:28.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 39408 39392 129.559 112.125 0.0115462 0.00771283 2024-09-14T04:25:28.843 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:28 smithi012 ceph-mon[101180]: pgmap v1366: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:25:28.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:28 smithi012 ceph-mon[101180]: osdmap e410: 8 total, 8 up, 6 in 2024-09-14T04:25:28.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:28.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:28 smithi001 ceph-mon[112165]: pgmap v1366: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:25:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:28 smithi001 ceph-mon[112165]: osdmap e410: 8 total, 8 up, 6 in 2024-09-14T04:25:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:28 smithi001 ceph-mon[107399]: pgmap v1366: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:25:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:28 smithi001 ceph-mon[107399]: osdmap e410: 8 total, 8 up, 6 in 2024-09-14T04:25:28.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:29.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:29.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:29.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:29.159 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:29.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:29.250 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:29.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:29.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:29.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:25:29.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:29.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:29.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:29.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:29.618 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:25:29.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[107399]: osdmap e411: 8 total, 8 up, 6 in 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/416804733' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[112165]: osdmap e411: 8 total, 8 up, 6 in 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/416804733' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:29 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:29.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:29.733 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:25:29.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:25:29.783975+0000 min lat: 0.00114104 max lat: 2.05783 avg lat: 0.00776097 2024-09-14T04:25:29.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:25:29.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 41228 41212 128.768 113.75 0.00658989 0.00776097 2024-09-14T04:25:29.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:29.864 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:29.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:29.919 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:29 smithi012 ceph-mon[101180]: osdmap e411: 8 total, 8 up, 6 in 2024-09-14T04:25:29.919 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/416804733' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.919 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:29 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:29.965 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:30.007 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:30.055 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:25:30.055 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:25:30.092 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:25:30.092 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:25:30.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:30.128 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:25:30.128 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:25:30.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:30.165 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:25:30.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:25:30.201 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:25:30.201 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:25:30.236 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:25:30.236 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:25:30.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:30.270 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:25:30.271 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:25:30.278 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:30.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:25:30.308 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:25:30.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:30.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:30.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:30.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[112165]: pgmap v1369: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:25:30.629 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:30.629 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[107399]: pgmap v1369: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:25:30.629 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:30.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:30.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:30.784 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 43042 43026 128.035 113.375 0.0116087 0.00780549 2024-09-14T04:25:30.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:30.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:30.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:30.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:30.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[107399]: osdmap e412: 8 total, 8 up, 6 in 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1432399680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[112165]: osdmap e412: 8 total, 8 up, 6 in 2024-09-14T04:25:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1432399680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:31.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:30 smithi012 ceph-mon[101180]: pgmap v1369: 109 pgs: 2 active+undersized+remapped, 1 peering, 106 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:25:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:30 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:30 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:30 smithi012 ceph-mon[101180]: osdmap e412: 8 total, 8 up, 6 in 2024-09-14T04:25:31.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1432399680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:31.108 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:31.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:31.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:31.359 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:31.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:31.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:31.537 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:31.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:31.675 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:31.776 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:31.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 44869 44853 127.403 114.188 0.00892553 0.00784332 2024-09-14T04:25:31.812 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:31.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:31.823 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:31 smithi012 ceph-mon[101180]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 107 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 87 MiB/s wr, 1.38k op/s; 2040/122169 objects misplaced (1.670%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:25:31.886 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:25:31.886 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:25:31.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:25:31.886 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:25:31.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:3450 13.b 13:d500c13c:::benchmark_data_smithi001_306460_object3449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e405)", 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:10.734442+0000", 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 21.148260046000001, 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014627017000000001, 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3450 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734442+0000", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734442+0000", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734441+0000", 2024-09-14T04:25:31.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734493+0000", 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1752999999999997e-05 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734502+0000", 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6059999999999992e-06 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734509+0000", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.063e-06 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734552+0000", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3120000000000001e-05 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734659+0000", 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106346 2024-09-14T04:25:31.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734851+0000", 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019170100000000001 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.735956+0000", 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011050299999999999 2024-09-14T04:25:31.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.736392+0000", 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043680100000000001 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.748982+0000", 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012589232000000001 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.749036+0000", 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4848000000000001e-05 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.749069+0000", 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2452e-05 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:3451 13.b 13:dfd5fe70:::benchmark_data_smithi001_306460_object3450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e405)", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:10.734519+0000", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 21.148182650999999, 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014661777000000001, 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3451 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734519+0000", 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734519+0000", 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734519+0000", 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734557+0000", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8671000000000003e-05 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734566+0000", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1079999999999999e-06 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.734572+0000", 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0270000000000001e-06 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.735471+0000", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00089811700000000003 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.735579+0000", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010861000000000001 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.735714+0000", 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013531599999999999 2024-09-14T04:25:31.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.736074+0000", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035970799999999998 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.736532+0000", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045815800000000001 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.749106+0000", 2024-09-14T04:25:31.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012573228000000001 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.749148+0000", 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1971999999999997e-05 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:10.749181+0000", 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3460000000000002e-05 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:42864 13.e 13:7c2f3916:::benchmark_data_smithi001_306460_object42863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:30.680278+0000", 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.2024242540000001, 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016518726000000001, 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.900 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 42864 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680278+0000", 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680278+0000", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680277+0000", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680569+0000", 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029168899999999998 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680583+0000", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4229e-05 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.680590+0000", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3309999999999998e-06 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.686644+0000", 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0060539280000000001 2024-09-14T04:25:31.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.686766+0000", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012216500000000001 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.686891+0000", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012510900000000001 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.690276+0000", 2024-09-14T04:25:31.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003384143 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.693942+0000", 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036664900000000001 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.696716+0000", 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027737059999999999 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.696762+0000", 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5686999999999998e-05 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.696796+0000", 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.485e-05 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:43035 13.b 13:dbcef9e1:::benchmark_data_smithi001_306460_object43034:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.906 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:30.784086+0000", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0986154939999999, 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015440924, 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 43035 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784086+0000", 2024-09-14T04:25:31.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784086+0000", 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784085+0000", 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784181+0000", 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5773000000000006e-05 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784195+0000", 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4433999999999999e-05 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784203+0000", 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4270000000000002e-06 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784234+0000", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1229999999999997e-05 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784336+0000", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101724 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784497+0000", 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000160705 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.787884+0000", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033870910000000001 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.792013+0000", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041295259999999997 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799463+0000", 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074501109999999997 2024-09-14T04:25:31.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799502+0000", 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8340999999999998e-05 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799527+0000", 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5698999999999999e-05 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:43043 13.b 13:dd794182:::benchmark_data_smithi001_306460_object43042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:30.784634+0000", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0980681859999999, 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015057956000000001, 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 43043 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784634+0000", 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784634+0000", 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784633+0000", 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784723+0000", 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9839999999999994e-05 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784733+0000", 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8740000000000007e-06 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.784739+0000", 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1680000000000001e-06 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.786090+0000", 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001351603 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.786230+0000", 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013949000000000001 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.786340+0000", 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110585 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.789104+0000", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027631859999999999 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.792052+0000", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029482499999999999 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799578+0000", 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075260730000000003 2024-09-14T04:25:31.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799635+0000", 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7250999999999998e-05 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:30.799692+0000", 2024-09-14T04:25:31.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6428999999999998e-05 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:43459 13.7 e412/408 13:ebe3e336:::benchmark_data_smithi001_306460_object43458:head v 412'2783, mlcod=405'305)", 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.018405+0000", 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.86429653100000003, 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016494353, 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.920 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018405+0000", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018405+0000", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018405+0000", 2024-09-14T04:25:31.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018432+0000", 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7070999999999999e-05 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018442+0000", 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8090000000000004e-06 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018447+0000", 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8839999999999999e-06 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018552+0000", 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104706 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018577+0000", 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5162999999999999e-05 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.034865+0000", 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016287881000000001 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.034900+0000", 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4579000000000002e-05 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.924 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:43472 13.7 e412/408 13:eb5e61d7:::benchmark_data_smithi001_306460_object43471:head v 412'2784, mlcod=405'305)", 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.018908+0000", 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.86379376399999996, 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016040563000000001, 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018908+0000", 2024-09-14T04:25:31.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018908+0000", 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018908+0000", 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018926+0000", 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7929999999999999e-05 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018932+0000", 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4280000000000001e-06 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.018936+0000", 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8689999999999997e-06 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.022464+0000", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00352812 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.022485+0000", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1141999999999998e-05 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.034924+0000", 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012439087999999999 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.034949+0000", 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4519e-05 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:43752 13.9 13:9b634645:::benchmark_data_smithi001_306460_object43751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.166929+0000", 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.71577305999999996, 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019635017000000001, 2024-09-14T04:25:31.929 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 43752 2024-09-14T04:25:31.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.166929+0000", 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.166929+0000", 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.166927+0000", 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.166988+0000", 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0887000000000002e-05 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.167002+0000", 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3814e-05 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.167009+0000", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4059999999999999e-06 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.167055+0000", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5649000000000003e-05 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.167168+0000", 2024-09-14T04:25:31.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011336399999999999 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.167287+0000", 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118794 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.169189+0000", 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019022450000000001 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.170969+0000", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001779978 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.186461+0000", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015491882 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.186522+0000", 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0470000000000002e-05 2024-09-14T04:25:31.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.186564+0000", 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2296000000000003e-05 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:43871 13.7 e412/408 13:e649aaa3:::benchmark_data_smithi001_306460_object43870:head v 412'2804, mlcod=405'321)", 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.240028+0000", 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.64267415299999997, 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022318747999999999, 2024-09-14T04:25:31.937 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240028+0000", 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240028+0000", 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240027+0000", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240057+0000", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0151e-05 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240072+0000", 2024-09-14T04:25:31.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5307000000000001e-05 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.240079+0000", 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.224e-06 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.243469+0000", 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033903890000000002 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.243494+0000", 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4576e-05 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262312+0000", 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018818714 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262346+0000", 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4136e-05 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.941 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:43875 13.7 e412/408 13:ec06ccef:::benchmark_data_smithi001_306460_object43874:head v 412'2805, mlcod=405'321)", 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.244969+0000", 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.63773292100000001, 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017411119999999999, 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.244969+0000", 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.244969+0000", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.244969+0000", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245004+0000", 2024-09-14T04:25:31.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5222000000000003e-05 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245012+0000", 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6960000000000005e-06 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245016+0000", 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.617e-06 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.246605+0000", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001588496 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.246630+0000", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5242e-05 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262352+0000", 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015722265999999999 2024-09-14T04:25:31.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262380+0000", 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7886e-05 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:43876 13.7 e412/408 13:ef1fc244:::benchmark_data_smithi001_306460_object43875:head v 412'2806, mlcod=405'321)", 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.245164+0000", 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.637537833, 2024-09-14T04:25:31.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017253602, 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245164+0000", 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245164+0000", 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245164+0000", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245185+0000", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0914e-05 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245192+0000", 2024-09-14T04:25:31.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.99e-06 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.245197+0000", 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7820000000000001e-06 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.252969+0000", 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007772195 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.252996+0000", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6964000000000001e-05 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262391+0000", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009395713 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.262418+0000", 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6284000000000001e-05 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.950 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:43972 13.9 13:9d728d74:::benchmark_data_smithi001_306460_object43971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.296222+0000", 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.58647954099999999, 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015461763999999999, 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 43972 2024-09-14T04:25:31.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296222+0000", 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296222+0000", 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296222+0000", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296254+0000", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2261000000000002e-05 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296264+0000", 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5380000000000008e-06 2024-09-14T04:25:31.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.296269+0000", 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0289999999999996e-06 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.298050+0000", 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001781443 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.298177+0000", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012686500000000001 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.298323+0000", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000145771 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.303808+0000", 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054857949999999999 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.303988+0000", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017911700000000001 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.311614+0000", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076265739999999997 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.311645+0000", 2024-09-14T04:25:31.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0929000000000002e-05 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.311684+0000", 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9001000000000001e-05 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:43976 13.9 13:9f7f9ac6:::benchmark_data_smithi001_306460_object43975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.299357+0000", 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.58334506799999997, 2024-09-14T04:25:31.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014994739999999999, 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 43976 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299357+0000", 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299357+0000", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299356+0000", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299388+0000", 2024-09-14T04:25:31.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1272000000000001e-05 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299396+0000", 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1950000000000003e-06 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.299401+0000", 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4029999999999998e-06 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.301661+0000", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002259732 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.301766+0000", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010563199999999999 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.301855+0000", 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8651000000000002e-05 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.304114+0000", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022588999999999999 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.306663+0000", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025488569999999999 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.314267+0000", 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076039799999999998 2024-09-14T04:25:31.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.314316+0000", 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8760000000000001e-05 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.314352+0000", 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5948000000000002e-05 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44016 13.9 13:9714e9b7:::benchmark_data_smithi001_306460_object44015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.323117+0000", 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.55958503000000004, 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016771732000000001, 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44016 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323117+0000", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323117+0000", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323116+0000", 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:25:31.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323165+0000", 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9499000000000002e-05 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323172+0000", 2024-09-14T04:25:31.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3050000000000004e-06 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323177+0000", 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1450000000000001e-06 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323205+0000", 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8101000000000001e-05 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323311+0000", 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106248 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.323509+0000", 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019771 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325494+0000", 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019850200000000001 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.329014+0000", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00351981 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.339820+0000", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010806754999999999 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.339852+0000", 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1272000000000001e-05 2024-09-14T04:25:31.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.339889+0000", 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7060000000000001e-05 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44021 13.9 13:918a617e:::benchmark_data_smithi001_306460_object44020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.324229+0000", 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.558473259, 2024-09-14T04:25:31.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017078296, 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44021 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324229+0000", 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324229+0000", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324228+0000", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324257+0000", 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9481000000000001e-05 2024-09-14T04:25:31.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324265+0000", 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2119999999999997e-06 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.324270+0000", 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.95e-06 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325521+0000", 2024-09-14T04:25:31.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012513369999999999 2024-09-14T04:25:31.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325638+0000", 2024-09-14T04:25:31.975 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:31 smithi001 ceph-mon[107399]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 107 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 87 MiB/s wr, 1.38k op/s; 2040/122169 objects misplaced (1.670%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:25:31.976 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:31 smithi001 ceph-mon[112165]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 107 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 87 MiB/s wr, 1.38k op/s; 2040/122169 objects misplaced (1.670%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011732500000000001 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325908+0000", 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027018700000000002 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.328981+0000", 2024-09-14T04:25:31.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030727480000000001 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.333503+0000", 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004522112 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.339938+0000", 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064352230000000003 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.340047+0000", 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108167 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.341307+0000", 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012603040000000001 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44024 13.9 13:91d5d7d9:::benchmark_data_smithi001_306460_object44023:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.978 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.325929+0000", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.55677241499999996, 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015544459, 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44024 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325929+0000", 2024-09-14T04:25:31.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325929+0000", 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325928+0000", 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325959+0000", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0502e-05 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325971+0000", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2507000000000001e-05 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.325978+0000", 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1709999999999996e-06 2024-09-14T04:25:31.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.327615+0000", 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001637051 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.327753+0000", 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013802899999999999 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.327889+0000", 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013603000000000001 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.329203+0000", 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001313315 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.333535+0000", 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043318799999999998 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.341374+0000", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0078389180000000003 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.341440+0000", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6174999999999996e-05 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.341474+0000", 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4233999999999998e-05 2024-09-14T04:25:31.984 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44392 13.e 13:7b00d6e6:::benchmark_data_smithi001_306460_object44391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.525045+0000", 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.357657012, 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014784904999999999, 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44392 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525045+0000", 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525045+0000", 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525044+0000", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525077+0000", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2354000000000002e-05 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525086+0000", 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9029999999999993e-06 2024-09-14T04:25:31.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525092+0000", 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0639999999999999e-06 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525563+0000", 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000471447 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525687+0000", 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012411 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.525819+0000", 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013142000000000001 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.527640+0000", 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018209159999999999 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.528883+0000", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001243734 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.539622+0000", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01073913 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.539710+0000", 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7225000000000006e-05 2024-09-14T04:25:31.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.539830+0000", 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120188 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44579 13.9 13:961b0246:::benchmark_data_smithi001_306460_object44578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.624142+0000", 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.25855983399999999, 2024-09-14T04:25:31.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015503404, 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44579 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624142+0000", 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624142+0000", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624141+0000", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624177+0000", 2024-09-14T04:25:31.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6622000000000003e-05 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624189+0000", 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2208000000000001e-05 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624196+0000", 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.63e-06 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:31.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624240+0000", 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3498000000000003e-05 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624315+0000", 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5803000000000006e-05 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.624393+0000", 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7380999999999997e-05 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.626084+0000", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001691028 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.628838+0000", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027547349999999999 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.637606+0000", 2024-09-14T04:25:31.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087676190000000008 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.637656+0000", 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0324000000000001e-05 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.639645+0000", 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019890680000000001 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:31.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:44939 13.9 13:9eca5ca5:::benchmark_data_smithi001_306460_object44938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.824270+0000", 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.058432338, 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015713451999999999, 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 44939 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.998 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824270+0000", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824270+0000", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824268+0000", 2024-09-14T04:25:31.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824300+0000", 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1745000000000002e-05 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824315+0000", 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4833e-05 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:32.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.824322+0000", 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.002e-06 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.825354+0000", 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001031959 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.825491+0000", 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137269 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.825643+0000", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151743 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.828398+0000", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002755019 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.828868+0000", 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046983499999999999 2024-09-14T04:25:32.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.839911+0000", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011043084 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.839951+0000", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0238999999999999e-05 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.839983+0000", 2024-09-14T04:25:32.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2029000000000001e-05 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:44989 13.7 e412/408 13:ecc8aeb5:::benchmark_data_smithi001_306460_object44988:head v 412'2883, mlcod=405'360)", 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:31.857149+0000", 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025552933999999999, 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015042405999999999, 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:25:32.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857149+0000", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857149+0000", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857148+0000", 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:25:32.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857184+0000", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5868999999999998e-05 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857196+0000", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1871e-05 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.857202+0000", 2024-09-14T04:25:32.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4459999999999998e-06 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.860048+0000", 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028458820000000001 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.860071+0000", 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2402e-05 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.872168+0000", 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012096936000000001 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:31.872191+0000", 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3849000000000002e-05 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:25:32.008 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:25:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:32.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:32.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:32.154 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:32.256 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:32.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:32.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:32.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:32.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:32.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:32.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:32.711 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:32.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 46634 46618 126.658 110.312 0.0152838 0.00788959 2024-09-14T04:25:32.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:32.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.857 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:32.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:32.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:32.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:32.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:33.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:33.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:33.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:33.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:25:33.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:33.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:33.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:33.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:33.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:33.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:33.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:33.581 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:33.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.684 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:33.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:33.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:33.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 48140 48124 125.302 94.125 0.009748 0.00797546 2024-09-14T04:25:33.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:33.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:33.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:25:33.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:33.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:33.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:34.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:25:34.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:34.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:34.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:34.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:34.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:34.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:34.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:34.557 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:25:34.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:34.643 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:34.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:34.768 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:34.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 49490 49474 123.665 84.375 0.00237389 0.00803176 2024-09-14T04:25:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:34 smithi012 ceph-mon[101180]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 2040/130383 objects misplaced (1.565%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:25:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:34 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1705453593' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:34 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.870 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:34.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:34.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:34.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[107399]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 2040/130383 objects misplaced (1.565%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1705453593' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[112165]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 2040/130383 objects misplaced (1.565%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1705453593' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:34 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:25:34.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:35.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:35.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:35.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:35.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:35.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:35.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:35.303 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:35.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:35.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:35.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:35.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:35.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:35.617 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:35.640 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:35.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:35.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:35.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 49490 49474 118.909 0 - 0.00803176 2024-09-14T04:25:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:35 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:25:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:35 smithi012 ceph-mon[101180]: osdmap e413: 8 total, 8 up, 6 in 2024-09-14T04:25:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:35 smithi012 ceph-mon[101180]: osdmap e414: 8 total, 8 up, 6 in 2024-09-14T04:25:35.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:35.888 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:35.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:35.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:35.907 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[112165]: osdmap e413: 8 total, 8 up, 6 in 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[112165]: osdmap e414: 8 total, 8 up, 6 in 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[107399]: osdmap e413: 8 total, 8 up, 6 in 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:35.908 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:35 smithi001 ceph-mon[107399]: osdmap e414: 8 total, 8 up, 6 in 2024-09-14T04:25:35.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:36.014 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:36.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:36.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:36.240 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:36.263 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:36.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:36.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:36.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:36.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:36.563 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:25:36.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:25:36.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:36.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:36.676 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:36.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:36.761 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:36.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 49490 49474 114.505 0 - 0.00803176 2024-09-14T04:25:36.791 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:36 smithi001 ceph-mon[107399]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 106 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 1595/137967 objects misplaced (1.156%); 6.2 MiB/s, 99 objects/s recovering 2024-09-14T04:25:36.791 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:36 smithi001 ceph-mon[107399]: osdmap e415: 8 total, 8 up, 6 in 2024-09-14T04:25:36.792 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:36 smithi001 ceph-mon[112165]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 106 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 1595/137967 objects misplaced (1.156%); 6.2 MiB/s, 99 objects/s recovering 2024-09-14T04:25:36.792 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:36 smithi001 ceph-mon[112165]: osdmap e415: 8 total, 8 up, 6 in 2024-09-14T04:25:36.792 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:36] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.51.0" 2024-09-14T04:25:36.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:36 smithi012 ceph-mon[101180]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 106 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 1595/137967 objects misplaced (1.156%); 6.2 MiB/s, 99 objects/s recovering 2024-09-14T04:25:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:36 smithi012 ceph-mon[101180]: osdmap e415: 8 total, 8 up, 6 in 2024-09-14T04:25:36.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:36.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:36.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:36.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:36.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:37.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:37.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:37.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:37.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:37.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:37.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:37.428 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:37.450 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:25:37.451 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:25:37.485 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:25:37.486 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:25:37.522 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:25:37.522 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:25:37.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:37.561 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:25:37.561 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:25:37.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:37.594 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:25:37.594 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:25:37.633 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:25:37.633 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:25:37.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:37 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/442627745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:37.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:37 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/442627745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:37.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:37.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:25:37.678 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:25:37.715 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:25:37.715 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:25:37.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:37.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:37.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 50475 50459 112.613 20.5208 0.00429355 0.00887364 2024-09-14T04:25:37.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:37 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/442627745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:37.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:37.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:37.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:37.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:38.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:38.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:38.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:38.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:38.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:38.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:38.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:38.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:38.584 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:38.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:38.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:38.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 51937 51921 111.881 91.375 0.00763431 0.00893228 2024-09-14T04:25:38.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:38 smithi012 ceph-mon[101180]: pgmap v1377: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s; 1595/148557 objects misplaced (1.074%); 4.6 MiB/s, 74 objects/s recovering 2024-09-14T04:25:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:38 smithi012 ceph-mon[101180]: osdmap e416: 8 total, 8 up, 6 in 2024-09-14T04:25:38.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:38.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:38.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:38 smithi001 ceph-mon[112165]: pgmap v1377: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s; 1595/148557 objects misplaced (1.074%); 4.6 MiB/s, 74 objects/s recovering 2024-09-14T04:25:38.939 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:38 smithi001 ceph-mon[112165]: osdmap e416: 8 total, 8 up, 6 in 2024-09-14T04:25:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:38 smithi001 ceph-mon[107399]: pgmap v1377: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s; 1595/148557 objects misplaced (1.074%); 4.6 MiB/s, 74 objects/s recovering 2024-09-14T04:25:38.939 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:38 smithi001 ceph-mon[107399]: osdmap e416: 8 total, 8 up, 6 in 2024-09-14T04:25:38.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:39.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:39.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.152 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:39.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.293 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:39.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:39.406 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:39.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:39.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:39.623 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:25:39.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:39.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:39.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:39.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 53447 53431 111.297 94.375 0.017755 0.00897952 2024-09-14T04:25:39.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:39 smithi012 ceph-mon[101180]: osdmap e417: 8 total, 8 up, 6 in 2024-09-14T04:25:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2309600014' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:39.879 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[107399]: osdmap e417: 8 total, 8 up, 6 in 2024-09-14T04:25:39.879 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2309600014' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.880 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.880 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[112165]: osdmap e417: 8 total, 8 up, 6 in 2024-09-14T04:25:39.880 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2309600014' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.880 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:25:39.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:39.990 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:39.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:40.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:40.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:40.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:40.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:40.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:40.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:40.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:40.533 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:40.638 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:40.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:40.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:40.785 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 54765 54749 110.364 82.375 0.0059544 0.00905544 2024-09-14T04:25:40.818 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:40.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[107399]: pgmap v1380: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 882 op/s; 1595/148557 objects misplaced (1.074%) 2024-09-14T04:25:40.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:40.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:25:40.885 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[107399]: osdmap e418: 8 total, 8 up, 6 in 2024-09-14T04:25:40.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[112165]: pgmap v1380: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 882 op/s; 1595/148557 objects misplaced (1.074%) 2024-09-14T04:25:40.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:40.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:25:40.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:40 smithi001 ceph-mon[112165]: osdmap e418: 8 total, 8 up, 6 in 2024-09-14T04:25:40.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:40.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:40.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:41.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:41.064 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:40 smithi012 ceph-mon[101180]: pgmap v1380: 109 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 882 op/s; 1595/148557 objects misplaced (1.074%) 2024-09-14T04:25:41.065 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:40 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:25:41.065 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:25:41.065 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:40 smithi012 ceph-mon[101180]: osdmap e418: 8 total, 8 up, 6 in 2024-09-14T04:25:41.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:41.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:41.232 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:41.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:41.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:41.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:25:41.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:41.500 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:41.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:41.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:41.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:41.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:41.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:41.786 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 56063 56047 109.45 81.125 0.0112462 0.00913004 2024-09-14T04:25:41.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:41.926 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:41.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:42.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:42.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:42.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:42.194 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:42.296 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:42.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:42.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:42.475 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:42.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:42.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:42.645 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:25:42.645 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:25:42.683 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:25:42.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:42 smithi001 ceph-mon[112165]: pgmap v1382: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s; 3569/158982 objects misplaced (2.245%); 14 MiB/s, 226 objects/s recovering 2024-09-14T04:25:42.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:42 smithi001 ceph-mon[107399]: pgmap v1382: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s; 3569/158982 objects misplaced (2.245%); 14 MiB/s, 226 objects/s recovering 2024-09-14T04:25:42.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:42.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:42.788 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 57258 57242 108.388 74.6875 0.00999542 0.0092187 2024-09-14T04:25:42.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:42.924 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:42.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:43.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:43.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:42 smithi012 ceph-mon[101180]: pgmap v1382: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s; 3569/158982 objects misplaced (2.245%); 14 MiB/s, 226 objects/s recovering 2024-09-14T04:25:43.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:43.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:43.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:43.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:43.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:43.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:43.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:43.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:43.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:43.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:43.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 58585 58569 107.639 82.9375 0.00660627 0.0092742 2024-09-14T04:25:43.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:43.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:43.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:43.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:43.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:43.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:44.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:25:44.096 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:44.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:44.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:44.244 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:44.315 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:44.315 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:44.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:44.372 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:44.474 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:44.497 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:44.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:44.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:44.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:44.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:44.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 58858 58842 105.052 17.0625 0.0106557 0.00951364 2024-09-14T04:25:44.856 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:25:44.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:44 smithi001 ceph-mon[107399]: pgmap v1383: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s; 3569/165360 objects misplaced (2.158%); 12 MiB/s, 186 objects/s recovering 2024-09-14T04:25:44.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:44 smithi001 ceph-mon[107399]: osdmap e419: 8 total, 8 up, 6 in 2024-09-14T04:25:44.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:44 smithi001 ceph-mon[112165]: pgmap v1383: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s; 3569/165360 objects misplaced (2.158%); 12 MiB/s, 186 objects/s recovering 2024-09-14T04:25:44.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:44 smithi001 ceph-mon[112165]: osdmap e419: 8 total, 8 up, 6 in 2024-09-14T04:25:44.859 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:25:44.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:44.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:44.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:44.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:45.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:44 smithi012 ceph-mon[101180]: pgmap v1383: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s; 3569/165360 objects misplaced (2.158%); 12 MiB/s, 186 objects/s recovering 2024-09-14T04:25:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:44 smithi012 ceph-mon[101180]: osdmap e419: 8 total, 8 up, 6 in 2024-09-14T04:25:45.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:45.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:45.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:45.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:45.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:45.399 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:45.440 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:45.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:45.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:45.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:45.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:45.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 60309 60293 104.653 90.6875 0.0079577 0.00954996 2024-09-14T04:25:45.820 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:45.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:45.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:45.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:45.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[107399]: osdmap e420: 8 total, 8 up, 6 in 2024-09-14T04:25:45.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[107399]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:25:45.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[107399]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 2951/170544 objects misplaced (1.730%); 18 MiB/s, 289 objects/s recovering 2024-09-14T04:25:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[112165]: osdmap e420: 8 total, 8 up, 6 in 2024-09-14T04:25:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[112165]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:25:45.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:45 smithi001 ceph-mon[112165]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 2951/170544 objects misplaced (1.730%); 18 MiB/s, 289 objects/s recovering 2024-09-14T04:25:46.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:46.070 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1385,"stamp":"2024-09-14T04:25:44.632135+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3635577016,"num_objects":55491,"num_object_clones":0,"num_object_copies":166473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3569,"num_objects_unfound":0,"num_objects_dirty":55491,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":55539,"num_write_kb":3549903,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1870,"num_bytes_recovered":120811771,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58653,"ondisk_log_size":58653,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15095776,"kb_used_data":10703584,"kb_used_omap":38,"kb_used_meta":4392089,"kb_avail":547375136,"statfs":{"total":575970213888,"available":560512139264,"internally_reserved":0,"allocated":10960470016,"data_stored":10944176677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39753,"internal_metadata":4497499319},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":391380992,"num_objects":5972,"num_object_clones":0,"num_object_copies":17916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1974,"num_objects_unfound":0,"num_objects_dirty":5972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1120,"num_bytes_recovered":73400320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.023878"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":641,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.639675+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:25:43.639675+0000","last_peered":"2024-09-14T04:25:43.639675+0000","last_clean":"2024-09-14T04:25:43.639675+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:25:43.639675+0000","last_undegraded":"2024-09-14T04:25:43.639675+0000","last_fullsized":"2024-09-14T04:25:43.639675+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":144,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.635213+0000","last_change":"2024-09-14T04:23:28.577777+0000","last_active":"2024-09-14T04:25:39.635213+0000","last_peered":"2024-09-14T04:25:39.635213+0000","last_clean":"2024-09-14T04:25:39.635213+0000","last_became_active":"2024-09-14T04:23:28.577155+0000","last_became_peered":"2024-09-14T04:23:28.577155+0000","last_unstale":"2024-09-14T04:25:39.635213+0000","last_undegraded":"2024-09-14T04:25:39.635213+0000","last_fullsized":"2024-09-14T04:25:39.635213+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:01.212541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":166,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.612202+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:25:38.612202+0000","last_peered":"2024-09-14T04:25:38.612202+0000","last_clean":"2024-09-14T04:25:38.612202+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:25:38.612202+0000","last_undegraded":"2024-09-14T04:25:38.612202+0000","last_fullsized":"2024-09-14T04:25:38.612202+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":756,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605424+0000","last_change":"2024-09-14T04:24:49.569053+0000","last_active":"2024-09-14T04:25:38.605424+0000","last_peered":"2024-09-14T04:25:38.605424+0000","last_clean":"2024-09-14T04:25:38.605424+0000","last_became_active":"2024-09-14T04:24:49.568690+0000","last_became_peered":"2024-09-14T04:24:49.568690+0000","last_unstale":"2024-09-14T04:25:38.605424+0000","last_undegraded":"2024-09-14T04:25:38.605424+0000","last_fullsized":"2024-09-14T04:25:38.605424+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:19:37.952706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605497+0000","last_change":"2024-09-14T04:25:25.582150+0000","last_active":"2024-09-14T04:25:38.605497+0000","last_peered":"2024-09-14T04:25:38.605497+0000","last_clean":"2024-09-14T04:25:38.605497+0000","last_became_active":"2024-09-14T04:25:25.581683+0000","last_became_peered":"2024-09-14T04:25:25.581683+0000","last_unstale":"2024-09-14T04:25:38.605497+0000","last_undegraded":"2024-09-14T04:25:38.605497+0000","last_fullsized":"2024-09-14T04:25:38.605497+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:10.188214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":457,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.639835+0000","last_change":"2024-09-14T04:25:35.562683+0000","last_active":"2024-09-14T04:25:43.639835+0000","last_peered":"2024-09-14T04:25:43.639835+0000","last_clean":"2024-09-14T04:25:43.639835+0000","last_became_active":"2024-09-14T04:25:35.562281+0000","last_became_peered":"2024-09-14T04:25:35.562281+0000","last_unstale":"2024-09-14T04:25:43.639835+0000","last_undegraded":"2024-09-14T04:25:43.639835+0000","last_fullsized":"2024-09-14T04:25:43.639835+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:09.493757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":928,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.630160+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:25:39.630160+0000","last_peered":"2024-09-14T04:25:39.630160+0000","last_clean":"2024-09-14T04:25:39.630160+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:25:39.630160+0000","last_undegraded":"2024-09-14T04:25:39.630160+0000","last_fullsized":"2024-09-14T04:25:39.630160+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":443,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643728+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:25:43.643728+0000","last_peered":"2024-09-14T04:25:43.643728+0000","last_clean":"2024-09-14T04:25:43.643728+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:25:43.643728+0000","last_undegraded":"2024-09-14T04:25:43.643728+0000","last_fullsized":"2024-09-14T04:25:43.643728+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":372,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610225+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:25:38.610225+0000","last_peered":"2024-09-14T04:25:38.610225+0000","last_clean":"2024-09-14T04:25:38.610225+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:25:38.610225+0000","last_undegraded":"2024-09-14T04:25:38.610225+0000","last_fullsized":"2024-09-14T04:25:38.610225+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":584,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605911+0000","last_change":"2024-09-14T04:24:49.572838+0000","last_active":"2024-09-14T04:25:38.605911+0000","last_peered":"2024-09-14T04:25:38.605911+0000","last_clean":"2024-09-14T04:25:38.605911+0000","last_became_active":"2024-09-14T04:24:49.572388+0000","last_became_peered":"2024-09-14T04:24:49.572388+0000","last_unstale":"2024-09-14T04:25:38.605911+0000","last_undegraded":"2024-09-14T04:25:38.605911+0000","last_fullsized":"2024-09-14T04:25:38.605911+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:22.578056+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.639199+0000","last_change":"2024-09-14T04:25:35.565538+0000","last_active":"2024-09-14T04:25:39.639199+0000","last_peered":"2024-09-14T04:25:39.639199+0000","last_clean":"2024-09-14T04:25:39.639199+0000","last_became_active":"2024-09-14T04:25:35.565210+0000","last_became_peered":"2024-09-14T04:25:35.565210+0000","last_unstale":"2024-09-14T04:25:39.639199+0000","last_undegraded":"2024-09-14T04:25:39.639199+0000","last_fullsized":"2024-09-14T04:25:39.639199+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:08.538693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":524,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640659+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:25:43.640659+0000","last_peered":"2024-09-14T04:25:43.640659+0000","last_clean":"2024-09-14T04:25:43.640659+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:25:43.640659+0000","last_undegraded":"2024-09-14T04:25:43.640659+0000","last_fullsized":"2024-09-14T04:25:43.640659+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"418'3455","reported_seq":3506,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:42.132969+0000","last_change":"2024-09-14T04:25:02.146250+0000","last_active":"2024-09-14T04:25:42.132969+0000","last_peered":"2024-09-14T04:25:42.132969+0000","last_clean":"2024-09-14T04:25:42.132969+0000","last_became_active":"2024-09-14T04:25:02.145730+0000","last_became_peered":"2024-09-14T04:25:02.145730+0000","last_unstale":"2024-09-14T04:25:42.132969+0000","last_undegraded":"2024-09-14T04:25:42.132969+0000","last_fullsized":"2024-09-14T04:25:42.132969+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3455,"log_dups_size":0,"ondisk_log_size":3455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:04.224581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226426880,"num_objects":3455,"num_object_clones":0,"num_object_copies":10365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3455,"num_write_kb":221120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":739,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611110+0000","last_change":"2024-09-14T04:24:49.572850+0000","last_active":"2024-09-14T04:25:38.611110+0000","last_peered":"2024-09-14T04:25:38.611110+0000","last_clean":"2024-09-14T04:25:38.611110+0000","last_became_active":"2024-09-14T04:24:49.572405+0000","last_became_peered":"2024-09-14T04:24:49.572405+0000","last_unstale":"2024-09-14T04:25:38.611110+0000","last_undegraded":"2024-09-14T04:25:38.611110+0000","last_fullsized":"2024-09-14T04:25:38.611110+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:06.709118+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":134,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611621+0000","last_change":"2024-09-14T04:23:28.575634+0000","last_active":"2024-09-14T04:25:38.611621+0000","last_peered":"2024-09-14T04:25:38.611621+0000","last_clean":"2024-09-14T04:25:38.611621+0000","last_became_active":"2024-09-14T04:23:28.558391+0000","last_became_peered":"2024-09-14T04:23:28.558391+0000","last_unstale":"2024-09-14T04:25:38.611621+0000","last_undegraded":"2024-09-14T04:25:38.611621+0000","last_fullsized":"2024-09-14T04:25:38.611621+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:03.845925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605202+0000","last_change":"2024-09-14T04:25:25.578143+0000","last_active":"2024-09-14T04:25:38.605202+0000","last_peered":"2024-09-14T04:25:38.605202+0000","last_clean":"2024-09-14T04:25:38.605202+0000","last_became_active":"2024-09-14T04:25:25.577784+0000","last_became_peered":"2024-09-14T04:25:25.577784+0000","last_unstale":"2024-09-14T04:25:38.605202+0000","last_undegraded":"2024-09-14T04:25:38.605202+0000","last_fullsized":"2024-09-14T04:25:38.605202+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:25.682719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"418'3484","reported_seq":3535,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:41.065689+0000","last_change":"2024-09-14T04:25:01.992464+0000","last_active":"2024-09-14T04:25:41.065689+0000","last_peered":"2024-09-14T04:25:41.065689+0000","last_clean":"2024-09-14T04:25:41.065689+0000","last_became_active":"2024-09-14T04:25:01.992117+0000","last_became_peered":"2024-09-14T04:25:01.992117+0000","last_unstale":"2024-09-14T04:25:41.065689+0000","last_undegraded":"2024-09-14T04:25:41.065689+0000","last_fullsized":"2024-09-14T04:25:41.065689+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3484,"log_dups_size":0,"ondisk_log_size":3484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:21.289671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228327424,"num_objects":3484,"num_object_clones":0,"num_object_copies":10452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3484,"num_write_kb":222976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":981,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.637798+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:25:39.637798+0000","last_peered":"2024-09-14T04:25:39.637798+0000","last_clean":"2024-09-14T04:25:39.637798+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:25:39.637798+0000","last_undegraded":"2024-09-14T04:25:39.637798+0000","last_fullsized":"2024-09-14T04:25:39.637798+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":31,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610858+0000","last_change":"2024-09-14T04:25:07.963635+0000","last_active":"2024-09-14T04:25:38.610858+0000","last_peered":"2024-09-14T04:25:38.610858+0000","last_clean":"2024-09-14T04:25:38.610858+0000","last_became_active":"2024-09-14T04:25:07.959540+0000","last_became_peered":"2024-09-14T04:25:07.959540+0000","last_unstale":"2024-09-14T04:25:38.610858+0000","last_undegraded":"2024-09-14T04:25:38.610858+0000","last_fullsized":"2024-09-14T04:25:38.610858+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:05.353685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":443,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610142+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:25:38.610142+0000","last_peered":"2024-09-14T04:25:38.610142+0000","last_clean":"2024-09-14T04:25:38.610142+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:25:38.610142+0000","last_undegraded":"2024-09-14T04:25:38.610142+0000","last_fullsized":"2024-09-14T04:25:38.610142+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"419'3695","reported_seq":3748,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.876104+0000","last_change":"2024-09-14T04:25:02.146419+0000","last_active":"2024-09-14T04:25:43.876104+0000","last_peered":"2024-09-14T04:25:43.876104+0000","last_clean":"2024-09-14T04:25:43.876104+0000","last_became_active":"2024-09-14T04:25:02.145877+0000","last_became_peered":"2024-09-14T04:25:02.145877+0000","last_unstale":"2024-09-14T04:25:43.876104+0000","last_undegraded":"2024-09-14T04:25:43.876104+0000","last_fullsized":"2024-09-14T04:25:43.876104+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3695,"log_dups_size":0,"ondisk_log_size":3695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:31.774348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242155520,"num_objects":3695,"num_object_clones":0,"num_object_copies":11085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3695,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":439,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.607396+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:25:38.607396+0000","last_peered":"2024-09-14T04:25:38.607396+0000","last_clean":"2024-09-14T04:25:38.607396+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:25:38.607396+0000","last_undegraded":"2024-09-14T04:25:38.607396+0000","last_fullsized":"2024-09-14T04:25:38.607396+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":25,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605383+0000","last_change":"2024-09-14T04:25:25.584570+0000","last_active":"2024-09-14T04:25:38.605383+0000","last_peered":"2024-09-14T04:25:38.605383+0000","last_clean":"2024-09-14T04:25:38.605383+0000","last_became_active":"2024-09-14T04:25:25.581721+0000","last_became_peered":"2024-09-14T04:25:25.581721+0000","last_unstale":"2024-09-14T04:25:38.605383+0000","last_undegraded":"2024-09-14T04:25:38.605383+0000","last_fullsized":"2024-09-14T04:25:38.605383+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:59.618866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":428,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629111+0000","last_change":"2024-09-14T04:24:08.985773+0000","last_active":"2024-09-14T04:25:39.629111+0000","last_peered":"2024-09-14T04:25:39.629111+0000","last_clean":"2024-09-14T04:25:39.629111+0000","last_became_active":"2024-09-14T04:24:08.985458+0000","last_became_peered":"2024-09-14T04:24:08.985458+0000","last_unstale":"2024-09-14T04:25:39.629111+0000","last_undegraded":"2024-09-14T04:25:39.629111+0000","last_fullsized":"2024-09-14T04:25:39.629111+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:10.470781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"418'3462","reported_seq":3791,"reported_epoch":418,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:25:41.003521+0000","last_change":"2024-09-14T04:25:37.062687+0000","last_active":"2024-09-14T04:25:41.003521+0000","last_peered":"2024-09-14T04:25:41.003521+0000","last_clean":"2024-09-14T04:25:34.547607+0000","last_became_active":"2024-09-14T04:25:37.041836+0000","last_became_peered":"2024-09-14T04:25:37.041836+0000","last_unstale":"2024-09-14T04:25:41.003521+0000","last_undegraded":"2024-09-14T04:25:41.003521+0000","last_fullsized":"2024-09-14T04:25:36.555651+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3462,"log_dups_size":0,"ondisk_log_size":3462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.438646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226885632,"num_objects":3462,"num_object_clones":0,"num_object_copies":10386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2951,"num_objects_unfound":0,"num_objects_dirty":3462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3462,"num_write_kb":221568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2951},{"shards":"4,5","objects":511}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":394,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610761+0000","last_change":"2024-09-14T04:25:10.210264+0000","last_active":"2024-09-14T04:25:38.610761+0000","last_peered":"2024-09-14T04:25:38.610761+0000","last_clean":"2024-09-14T04:25:38.610761+0000","last_became_active":"2024-09-14T04:25:10.209992+0000","last_became_peered":"2024-09-14T04:25:10.209992+0000","last_unstale":"2024-09-14T04:25:38.610761+0000","last_undegraded":"2024-09-14T04:25:38.610761+0000","last_fullsized":"2024-09-14T04:25:38.610761+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:50.222369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:50.222369+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":366,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605990+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:25:38.605990+0000","last_peered":"2024-09-14T04:25:38.605990+0000","last_clean":"2024-09-14T04:25:38.605990+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:25:38.605990+0000","last_undegraded":"2024-09-14T04:25:38.605990+0000","last_fullsized":"2024-09-14T04:25:38.605990+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":436,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.646092+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:25:43.646092+0000","last_peered":"2024-09-14T04:25:43.646092+0000","last_clean":"2024-09-14T04:25:43.646092+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:25:43.646092+0000","last_undegraded":"2024-09-14T04:25:43.646092+0000","last_fullsized":"2024-09-14T04:25:43.646092+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"418'3429","reported_seq":3458,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.021247+0000","last_change":"2024-09-14T04:25:01.992668+0000","last_active":"2024-09-14T04:25:40.021247+0000","last_peered":"2024-09-14T04:25:40.021247+0000","last_clean":"2024-09-14T04:25:40.021247+0000","last_became_active":"2024-09-14T04:25:01.992359+0000","last_became_peered":"2024-09-14T04:25:01.992359+0000","last_unstale":"2024-09-14T04:25:40.021247+0000","last_undegraded":"2024-09-14T04:25:40.021247+0000","last_fullsized":"2024-09-14T04:25:40.021247+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3429,"log_dups_size":0,"ondisk_log_size":3429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:08.434277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224722944,"num_objects":3429,"num_object_clones":0,"num_object_copies":10287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3429,"num_write_kb":219456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":722,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631011+0000","last_change":"2024-09-14T04:24:10.193742+0000","last_active":"2024-09-14T04:25:39.631011+0000","last_peered":"2024-09-14T04:25:39.631011+0000","last_clean":"2024-09-14T04:25:39.631011+0000","last_became_active":"2024-09-14T04:24:10.193277+0000","last_became_peered":"2024-09-14T04:24:10.193277+0000","last_unstale":"2024-09-14T04:25:39.631011+0000","last_undegraded":"2024-09-14T04:25:39.631011+0000","last_fullsized":"2024-09-14T04:25:39.631011+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:08.270894+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":440,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.645731+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:25:43.645731+0000","last_peered":"2024-09-14T04:25:43.645731+0000","last_clean":"2024-09-14T04:25:43.645731+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:25:43.645731+0000","last_undegraded":"2024-09-14T04:25:43.645731+0000","last_fullsized":"2024-09-14T04:25:43.645731+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":419,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610746+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:25:38.610746+0000","last_peered":"2024-09-14T04:25:38.610746+0000","last_clean":"2024-09-14T04:25:38.610746+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:25:38.610746+0000","last_undegraded":"2024-09-14T04:25:38.610746+0000","last_fullsized":"2024-09-14T04:25:38.610746+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"418'3604","reported_seq":3655,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:42.147732+0000","last_change":"2024-09-14T04:25:01.995435+0000","last_active":"2024-09-14T04:25:42.147732+0000","last_peered":"2024-09-14T04:25:42.147732+0000","last_clean":"2024-09-14T04:25:42.147732+0000","last_became_active":"2024-09-14T04:25:01.995177+0000","last_became_peered":"2024-09-14T04:25:01.995177+0000","last_unstale":"2024-09-14T04:25:42.147732+0000","last_undegraded":"2024-09-14T04:25:42.147732+0000","last_fullsized":"2024-09-14T04:25:42.147732+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3604,"log_dups_size":0,"ondisk_log_size":3604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:02.315254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236191744,"num_objects":3604,"num_object_clones":0,"num_object_copies":10812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3604,"num_write_kb":230656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":652,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605326+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:25:38.605326+0000","last_peered":"2024-09-14T04:25:38.605326+0000","last_clean":"2024-09-14T04:25:38.605326+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:25:38.605326+0000","last_undegraded":"2024-09-14T04:25:38.605326+0000","last_fullsized":"2024-09-14T04:25:38.605326+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":518,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640108+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:25:43.640108+0000","last_peered":"2024-09-14T04:25:43.640108+0000","last_clean":"2024-09-14T04:25:43.640108+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:25:43.640108+0000","last_undegraded":"2024-09-14T04:25:43.640108+0000","last_fullsized":"2024-09-14T04:25:43.640108+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611246+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:25:38.611246+0000","last_peered":"2024-09-14T04:25:38.611246+0000","last_clean":"2024-09-14T04:25:38.611246+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:25:38.611246+0000","last_undegraded":"2024-09-14T04:25:38.611246+0000","last_fullsized":"2024-09-14T04:25:38.611246+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"418'3458","reported_seq":6818,"reported_epoch":418,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:25:40.060313+0000","last_change":"2024-09-14T04:25:26.654873+0000","last_active":"2024-09-14T04:25:40.060313+0000","last_peered":"2024-09-14T04:25:40.060313+0000","last_clean":"2024-09-14T04:25:24.552609+0000","last_became_active":"2024-09-14T04:25:26.638065+0000","last_became_peered":"2024-09-14T04:25:26.638065+0000","last_unstale":"2024-09-14T04:25:40.060313+0000","last_undegraded":"2024-09-14T04:25:40.060313+0000","last_fullsized":"2024-09-14T04:25:26.578082+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3458,"log_dups_size":0,"ondisk_log_size":3458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:33:47.682976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226623488,"num_objects":3458,"num_object_clones":0,"num_object_copies":10374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":618,"num_objects_unfound":0,"num_objects_dirty":3458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3458,"num_write_kb":221312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1662,"num_bytes_recovered":108920832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":618},{"shards":"0,4","objects":2840}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":908,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629341+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:25:39.629341+0000","last_peered":"2024-09-14T04:25:39.629341+0000","last_clean":"2024-09-14T04:25:39.629341+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:25:39.629341+0000","last_undegraded":"2024-09-14T04:25:39.629341+0000","last_fullsized":"2024-09-14T04:25:39.629341+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":27,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.609944+0000","last_change":"2024-09-14T04:25:07.967648+0000","last_active":"2024-09-14T04:25:38.609944+0000","last_peered":"2024-09-14T04:25:38.609944+0000","last_clean":"2024-09-14T04:25:38.609944+0000","last_became_active":"2024-09-14T04:25:07.967284+0000","last_became_peered":"2024-09-14T04:25:07.967284+0000","last_unstale":"2024-09-14T04:25:38.609944+0000","last_undegraded":"2024-09-14T04:25:38.609944+0000","last_fullsized":"2024-09-14T04:25:38.609944+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:29.967190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":425,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611080+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:25:38.611080+0000","last_peered":"2024-09-14T04:25:38.611080+0000","last_clean":"2024-09-14T04:25:38.611080+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:25:38.611080+0000","last_undegraded":"2024-09-14T04:25:38.611080+0000","last_fullsized":"2024-09-14T04:25:38.611080+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"418'3386","reported_seq":3415,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.056889+0000","last_change":"2024-09-14T04:25:02.146105+0000","last_active":"2024-09-14T04:25:40.056889+0000","last_peered":"2024-09-14T04:25:40.056889+0000","last_clean":"2024-09-14T04:25:40.056889+0000","last_became_active":"2024-09-14T04:25:02.145736+0000","last_became_peered":"2024-09-14T04:25:02.145736+0000","last_unstale":"2024-09-14T04:25:40.056889+0000","last_undegraded":"2024-09-14T04:25:40.056889+0000","last_fullsized":"2024-09-14T04:25:40.056889+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3386,"log_dups_size":0,"ondisk_log_size":3386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.907164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221904896,"num_objects":3386,"num_object_clones":0,"num_object_copies":10158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3386,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":622,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.606079+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:25:38.606079+0000","last_peered":"2024-09-14T04:25:38.606079+0000","last_clean":"2024-09-14T04:25:38.606079+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:25:38.606079+0000","last_undegraded":"2024-09-14T04:25:38.606079+0000","last_fullsized":"2024-09-14T04:25:38.606079+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631176+0000","last_change":"2024-09-14T04:25:35.561629+0000","last_active":"2024-09-14T04:25:39.631176+0000","last_peered":"2024-09-14T04:25:39.631176+0000","last_clean":"2024-09-14T04:25:39.631176+0000","last_became_active":"2024-09-14T04:25:35.561305+0000","last_became_peered":"2024-09-14T04:25:35.561305+0000","last_unstale":"2024-09-14T04:25:39.631176+0000","last_undegraded":"2024-09-14T04:25:39.631176+0000","last_fullsized":"2024-09-14T04:25:39.631176+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:39.574054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":678,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.630008+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:25:39.630008+0000","last_peered":"2024-09-14T04:25:39.630008+0000","last_clean":"2024-09-14T04:25:39.630008+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:25:39.630008+0000","last_undegraded":"2024-09-14T04:25:39.630008+0000","last_fullsized":"2024-09-14T04:25:39.630008+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"418'3536","reported_seq":3587,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:41.054846+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:25:41.054846+0000","last_peered":"2024-09-14T04:25:41.054846+0000","last_clean":"2024-09-14T04:25:41.054846+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:25:41.054846+0000","last_undegraded":"2024-09-14T04:25:41.054846+0000","last_fullsized":"2024-09-14T04:25:41.054846+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3536,"log_dups_size":0,"ondisk_log_size":3536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:09.181654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231735296,"num_objects":3536,"num_object_clones":0,"num_object_copies":10608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3536,"num_write_kb":226304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":779,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640816+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:25:43.640816+0000","last_peered":"2024-09-14T04:25:43.640816+0000","last_clean":"2024-09-14T04:25:43.640816+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:25:43.640816+0000","last_undegraded":"2024-09-14T04:25:43.640816+0000","last_fullsized":"2024-09-14T04:25:43.640816+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:17.530093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":356,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.606334+0000","last_change":"2024-09-14T04:24:08.979706+0000","last_active":"2024-09-14T04:25:38.606334+0000","last_peered":"2024-09-14T04:25:38.606334+0000","last_clean":"2024-09-14T04:25:38.606334+0000","last_became_active":"2024-09-14T04:24:08.979397+0000","last_became_peered":"2024-09-14T04:24:08.979397+0000","last_unstale":"2024-09-14T04:25:38.606334+0000","last_undegraded":"2024-09-14T04:25:38.606334+0000","last_fullsized":"2024-09-14T04:25:38.606334+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:12.232467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":310,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.606123+0000","last_change":"2024-09-14T04:24:07.976836+0000","last_active":"2024-09-14T04:25:38.606123+0000","last_peered":"2024-09-14T04:25:38.606123+0000","last_clean":"2024-09-14T04:25:38.606123+0000","last_became_active":"2024-09-14T04:24:07.976471+0000","last_became_peered":"2024-09-14T04:24:07.976471+0000","last_unstale":"2024-09-14T04:25:38.606123+0000","last_undegraded":"2024-09-14T04:25:38.606123+0000","last_fullsized":"2024-09-14T04:25:38.606123+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:20.472847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":222,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611218+0000","last_change":"2024-09-14T04:25:35.560355+0000","last_active":"2024-09-14T04:25:38.611218+0000","last_peered":"2024-09-14T04:25:38.611218+0000","last_clean":"2024-09-14T04:25:38.611218+0000","last_became_active":"2024-09-14T04:25:35.559975+0000","last_became_peered":"2024-09-14T04:25:35.559975+0000","last_unstale":"2024-09-14T04:25:38.611218+0000","last_undegraded":"2024-09-14T04:25:38.611218+0000","last_fullsized":"2024-09-14T04:25:38.611218+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:58.033714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"418'3459","reported_seq":3516,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:42.124672+0000","last_change":"2024-09-14T04:25:07.966414+0000","last_active":"2024-09-14T04:25:42.124672+0000","last_peered":"2024-09-14T04:25:42.124672+0000","last_clean":"2024-09-14T04:25:42.124672+0000","last_became_active":"2024-09-14T04:25:07.966014+0000","last_became_peered":"2024-09-14T04:25:07.966014+0000","last_unstale":"2024-09-14T04:25:42.124672+0000","last_undegraded":"2024-09-14T04:25:42.124672+0000","last_fullsized":"2024-09-14T04:25:42.124672+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3459,"log_dups_size":0,"ondisk_log_size":3459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:58.062778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226689024,"num_objects":3459,"num_object_clones":0,"num_object_copies":10377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3459,"num_write_kb":221376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":900,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629381+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:25:39.629381+0000","last_peered":"2024-09-14T04:25:39.629381+0000","last_clean":"2024-09-14T04:25:39.629381+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:25:39.629381+0000","last_undegraded":"2024-09-14T04:25:39.629381+0000","last_fullsized":"2024-09-14T04:25:39.629381+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":378,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629278+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:25:39.629278+0000","last_peered":"2024-09-14T04:25:39.629278+0000","last_clean":"2024-09-14T04:25:39.629278+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:25:39.629278+0000","last_undegraded":"2024-09-14T04:25:39.629278+0000","last_fullsized":"2024-09-14T04:25:39.629278+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":686,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.639579+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:25:43.639579+0000","last_peered":"2024-09-14T04:25:43.639579+0000","last_clean":"2024-09-14T04:25:43.639579+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:25:43.639579+0000","last_undegraded":"2024-09-14T04:25:43.639579+0000","last_fullsized":"2024-09-14T04:25:43.639579+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":447,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643775+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:25:43.643775+0000","last_peered":"2024-09-14T04:25:43.643775+0000","last_clean":"2024-09-14T04:25:43.643775+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:25:43.643775+0000","last_undegraded":"2024-09-14T04:25:43.643775+0000","last_fullsized":"2024-09-14T04:25:43.643775+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"418'3425","reported_seq":3454,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.946097+0000","last_change":"2024-09-14T04:25:01.992854+0000","last_active":"2024-09-14T04:25:40.946097+0000","last_peered":"2024-09-14T04:25:40.946097+0000","last_clean":"2024-09-14T04:25:40.946097+0000","last_became_active":"2024-09-14T04:25:01.992514+0000","last_became_peered":"2024-09-14T04:25:01.992514+0000","last_unstale":"2024-09-14T04:25:40.946097+0000","last_undegraded":"2024-09-14T04:25:40.946097+0000","last_fullsized":"2024-09-14T04:25:40.946097+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3425,"log_dups_size":0,"ondisk_log_size":3425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:59.789126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224460800,"num_objects":3425,"num_object_clones":0,"num_object_copies":10275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3425,"num_write_kb":219200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":654,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.639977+0000","last_change":"2024-09-14T04:25:10.213700+0000","last_active":"2024-09-14T04:25:43.639977+0000","last_peered":"2024-09-14T04:25:43.639977+0000","last_clean":"2024-09-14T04:25:43.639977+0000","last_became_active":"2024-09-14T04:25:10.213291+0000","last_became_peered":"2024-09-14T04:25:10.213291+0000","last_unstale":"2024-09-14T04:25:43.639977+0000","last_undegraded":"2024-09-14T04:25:43.639977+0000","last_fullsized":"2024-09-14T04:25:43.639977+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:58.394641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":651,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631202+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:25:39.631202+0000","last_peered":"2024-09-14T04:25:39.631202+0000","last_clean":"2024-09-14T04:25:39.631202+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:25:39.631202+0000","last_undegraded":"2024-09-14T04:25:39.631202+0000","last_fullsized":"2024-09-14T04:25:39.631202+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":682,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629427+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:25:39.629427+0000","last_peered":"2024-09-14T04:25:39.629427+0000","last_clean":"2024-09-14T04:25:39.629427+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:25:39.629427+0000","last_undegraded":"2024-09-14T04:25:39.629427+0000","last_fullsized":"2024-09-14T04:25:39.629427+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629942+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:25:39.629942+0000","last_peered":"2024-09-14T04:25:39.629942+0000","last_clean":"2024-09-14T04:25:39.629942+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:25:39.629942+0000","last_undegraded":"2024-09-14T04:25:39.629942+0000","last_fullsized":"2024-09-14T04:25:39.629942+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"418'3362","reported_seq":3391,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.024474+0000","last_change":"2024-09-14T04:25:02.146113+0000","last_active":"2024-09-14T04:25:40.024474+0000","last_peered":"2024-09-14T04:25:40.024474+0000","last_clean":"2024-09-14T04:25:40.024474+0000","last_became_active":"2024-09-14T04:25:02.145762+0000","last_became_peered":"2024-09-14T04:25:02.145762+0000","last_unstale":"2024-09-14T04:25:40.024474+0000","last_undegraded":"2024-09-14T04:25:40.024474+0000","last_fullsized":"2024-09-14T04:25:40.024474+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3362,"log_dups_size":0,"ondisk_log_size":3362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:33.000554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220332032,"num_objects":3362,"num_object_clones":0,"num_object_copies":10086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3362,"num_write_kb":215168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":645,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605571+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:25:38.605571+0000","last_peered":"2024-09-14T04:25:38.605571+0000","last_clean":"2024-09-14T04:25:38.605571+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:25:38.605571+0000","last_undegraded":"2024-09-14T04:25:38.605571+0000","last_fullsized":"2024-09-14T04:25:38.605571+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":353,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605997+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:25:38.605997+0000","last_peered":"2024-09-14T04:25:38.605997+0000","last_clean":"2024-09-14T04:25:38.605997+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:25:38.605997+0000","last_undegraded":"2024-09-14T04:25:38.605997+0000","last_fullsized":"2024-09-14T04:25:38.605997+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":436,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.612193+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:25:38.612193+0000","last_peered":"2024-09-14T04:25:38.612193+0000","last_clean":"2024-09-14T04:25:38.612193+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:25:38.612193+0000","last_undegraded":"2024-09-14T04:25:38.612193+0000","last_fullsized":"2024-09-14T04:25:38.612193+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":373,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605993+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:25:38.605993+0000","last_peered":"2024-09-14T04:25:38.605993+0000","last_clean":"2024-09-14T04:25:38.605993+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:25:38.605993+0000","last_undegraded":"2024-09-14T04:25:38.605993+0000","last_fullsized":"2024-09-14T04:25:38.605993+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"418'3476","reported_seq":3505,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.983345+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:25:40.983345+0000","last_peered":"2024-09-14T04:25:40.983345+0000","last_clean":"2024-09-14T04:25:40.983345+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:25:40.983345+0000","last_undegraded":"2024-09-14T04:25:40.983345+0000","last_fullsized":"2024-09-14T04:25:40.983345+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3476,"log_dups_size":0,"ondisk_log_size":3476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227803136,"num_objects":3476,"num_object_clones":0,"num_object_copies":10428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3476,"num_write_kb":222464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":672,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.639141+0000","last_change":"2024-09-14T04:25:38.633170+0000","last_active":"2024-09-14T04:25:39.639141+0000","last_peered":"2024-09-14T04:25:39.639141+0000","last_clean":"2024-09-14T04:25:39.639141+0000","last_became_active":"2024-09-14T04:25:38.632816+0000","last_became_peered":"2024-09-14T04:25:38.632816+0000","last_unstale":"2024-09-14T04:25:39.639141+0000","last_undegraded":"2024-09-14T04:25:39.639141+0000","last_fullsized":"2024-09-14T04:25:39.639141+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:55:39.218376+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":382,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605496+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:25:38.605496+0000","last_peered":"2024-09-14T04:25:38.605496+0000","last_clean":"2024-09-14T04:25:38.605496+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:25:38.605496+0000","last_undegraded":"2024-09-14T04:25:38.605496+0000","last_fullsized":"2024-09-14T04:25:38.605496+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":740,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.635541+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:25:39.635541+0000","last_peered":"2024-09-14T04:25:39.635541+0000","last_clean":"2024-09-14T04:25:39.635541+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:25:39.635541+0000","last_undegraded":"2024-09-14T04:25:39.635541+0000","last_fullsized":"2024-09-14T04:25:39.635541+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":166,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.629020+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:25:39.629020+0000","last_peered":"2024-09-14T04:25:39.629020+0000","last_clean":"2024-09-14T04:25:39.629020+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:25:39.629020+0000","last_undegraded":"2024-09-14T04:25:39.629020+0000","last_fullsized":"2024-09-14T04:25:39.629020+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"418'3460","reported_seq":3511,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:42.148867+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:25:42.148867+0000","last_peered":"2024-09-14T04:25:42.148867+0000","last_clean":"2024-09-14T04:25:42.148867+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:25:42.148867+0000","last_undegraded":"2024-09-14T04:25:42.148867+0000","last_fullsized":"2024-09-14T04:25:42.148867+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3460,"log_dups_size":0,"ondisk_log_size":3460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:13.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":221440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":691,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643576+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:25:43.643576+0000","last_peered":"2024-09-14T04:25:43.643576+0000","last_clean":"2024-09-14T04:25:43.643576+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:25:43.643576+0000","last_undegraded":"2024-09-14T04:25:43.643576+0000","last_fullsized":"2024-09-14T04:25:43.643576+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":686,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.635462+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:25:39.635462+0000","last_peered":"2024-09-14T04:25:39.635462+0000","last_clean":"2024-09-14T04:25:39.635462+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:25:39.635462+0000","last_undegraded":"2024-09-14T04:25:39.635462+0000","last_fullsized":"2024-09-14T04:25:39.635462+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":419,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643386+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:25:43.643386+0000","last_peered":"2024-09-14T04:25:43.643386+0000","last_clean":"2024-09-14T04:25:43.643386+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:25:43.643386+0000","last_undegraded":"2024-09-14T04:25:43.643386+0000","last_fullsized":"2024-09-14T04:25:43.643386+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":453,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.645949+0000","last_change":"2024-09-14T04:24:08.989110+0000","last_active":"2024-09-14T04:25:43.645949+0000","last_peered":"2024-09-14T04:25:43.645949+0000","last_clean":"2024-09-14T04:25:43.645949+0000","last_became_active":"2024-09-14T04:24:08.987440+0000","last_became_peered":"2024-09-14T04:24:08.987440+0000","last_unstale":"2024-09-14T04:25:43.645949+0000","last_undegraded":"2024-09-14T04:25:43.645949+0000","last_fullsized":"2024-09-14T04:25:43.645949+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:17:32.812292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"418'3348","reported_seq":3377,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:40.969338+0000","last_change":"2024-09-14T04:25:01.993057+0000","last_active":"2024-09-14T04:25:40.969338+0000","last_peered":"2024-09-14T04:25:40.969338+0000","last_clean":"2024-09-14T04:25:40.969338+0000","last_became_active":"2024-09-14T04:25:01.992668+0000","last_became_peered":"2024-09-14T04:25:01.992668+0000","last_unstale":"2024-09-14T04:25:40.969338+0000","last_undegraded":"2024-09-14T04:25:40.969338+0000","last_fullsized":"2024-09-14T04:25:40.969338+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3348,"log_dups_size":0,"ondisk_log_size":3348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:38.544577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219414528,"num_objects":3348,"num_object_clones":0,"num_object_copies":10044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3348,"num_write_kb":214272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":723,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.611084+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:25:38.611084+0000","last_peered":"2024-09-14T04:25:38.611084+0000","last_clean":"2024-09-14T04:25:38.611084+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:25:38.611084+0000","last_undegraded":"2024-09-14T04:25:38.611084+0000","last_fullsized":"2024-09-14T04:25:38.611084+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":342,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.648912+0000","last_change":"2024-09-14T04:24:08.999455+0000","last_active":"2024-09-14T04:25:43.648912+0000","last_peered":"2024-09-14T04:25:43.648912+0000","last_clean":"2024-09-14T04:25:43.648912+0000","last_became_active":"2024-09-14T04:24:08.994093+0000","last_became_peered":"2024-09-14T04:24:08.994093+0000","last_unstale":"2024-09-14T04:25:43.648912+0000","last_undegraded":"2024-09-14T04:25:43.648912+0000","last_fullsized":"2024-09-14T04:25:43.648912+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:29:31.948482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":671,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.630085+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:25:39.630085+0000","last_peered":"2024-09-14T04:25:39.630085+0000","last_clean":"2024-09-14T04:25:39.630085+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:25:39.630085+0000","last_undegraded":"2024-09-14T04:25:39.630085+0000","last_fullsized":"2024-09-14T04:25:39.630085+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":372,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640568+0000","last_change":"2024-09-14T04:25:35.562908+0000","last_active":"2024-09-14T04:25:43.640568+0000","last_peered":"2024-09-14T04:25:43.640568+0000","last_clean":"2024-09-14T04:25:43.640568+0000","last_became_active":"2024-09-14T04:25:35.562023+0000","last_became_peered":"2024-09-14T04:25:35.562023+0000","last_unstale":"2024-09-14T04:25:43.640568+0000","last_undegraded":"2024-09-14T04:25:43.640568+0000","last_fullsized":"2024-09-14T04:25:43.640568+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:55.656734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.8","version":"418'3426","reported_seq":3477,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:42.130904+0000","last_change":"2024-09-14T04:25:01.996105+0000","last_active":"2024-09-14T04:25:42.130904+0000","last_peered":"2024-09-14T04:25:42.130904+0000","last_clean":"2024-09-14T04:25:42.130904+0000","last_became_active":"2024-09-14T04:25:01.995811+0000","last_became_peered":"2024-09-14T04:25:01.995811+0000","last_unstale":"2024-09-14T04:25:42.130904+0000","last_undegraded":"2024-09-14T04:25:42.130904+0000","last_fullsized":"2024-09-14T04:25:42.130904+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":3426,"log_dups_size":0,"ondisk_log_size":3426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:04.759721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224526336,"num_objects":3426,"num_object_clones":0,"num_object_copies":10278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3426,"num_write_kb":219264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":961,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.637697+0000","last_change":"2024-09-14T04:25:28.605221+0000","last_active":"2024-09-14T04:25:39.637697+0000","last_peered":"2024-09-14T04:25:39.637697+0000","last_clean":"2024-09-14T04:25:39.637697+0000","last_became_active":"2024-09-14T04:25:28.604819+0000","last_became_peered":"2024-09-14T04:25:28.604819+0000","last_unstale":"2024-09-14T04:25:39.637697+0000","last_undegraded":"2024-09-14T04:25:39.637697+0000","last_fullsized":"2024-09-14T04:25:39.637697+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:21.033138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":440,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643679+0000","last_change":"2024-09-14T04:10:35.393494+0000","last_active":"2024-09-14T04:25:43.643679+0000","last_peered":"2024-09-14T04:25:43.643679+0000","last_clean":"2024-09-14T04:25:43.643679+0000","last_became_active":"2024-09-14T04:10:35.381969+0000","last_became_peered":"2024-09-14T04:10:35.381969+0000","last_unstale":"2024-09-14T04:25:43.643679+0000","last_undegraded":"2024-09-14T04:25:43.643679+0000","last_fullsized":"2024-09-14T04:25:43.643679+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:28.394604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":516,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631996+0000","last_change":"2024-09-14T04:23:27.596032+0000","last_active":"2024-09-14T04:25:39.631996+0000","last_peered":"2024-09-14T04:25:39.631996+0000","last_clean":"2024-09-14T04:25:39.631996+0000","last_became_active":"2024-09-14T04:23:27.595521+0000","last_became_peered":"2024-09-14T04:23:27.595521+0000","last_unstale":"2024-09-14T04:25:39.631996+0000","last_undegraded":"2024-09-14T04:25:39.631996+0000","last_fullsized":"2024-09-14T04:25:39.631996+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:14.145527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":402,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610156+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:25:38.610156+0000","last_peered":"2024-09-14T04:25:38.610156+0000","last_clean":"2024-09-14T04:25:38.610156+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:25:38.610156+0000","last_undegraded":"2024-09-14T04:25:38.610156+0000","last_fullsized":"2024-09-14T04:25:38.610156+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":439,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.612104+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:25:38.612104+0000","last_peered":"2024-09-14T04:25:38.612104+0000","last_clean":"2024-09-14T04:25:38.612104+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:25:38.612104+0000","last_undegraded":"2024-09-14T04:25:38.612104+0000","last_fullsized":"2024-09-14T04:25:38.612104+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":438,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610815+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:25:38.610815+0000","last_peered":"2024-09-14T04:25:38.610815+0000","last_clean":"2024-09-14T04:25:38.610815+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:25:38.610815+0000","last_undegraded":"2024-09-14T04:25:38.610815+0000","last_fullsized":"2024-09-14T04:25:38.610815+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.628792+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:25:39.628792+0000","last_peered":"2024-09-14T04:25:39.628792+0000","last_clean":"2024-09-14T04:25:39.628792+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:25:39.628792+0000","last_undegraded":"2024-09-14T04:25:39.628792+0000","last_fullsized":"2024-09-14T04:25:39.628792+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":355,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.638882+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:25:39.638882+0000","last_peered":"2024-09-14T04:25:39.638882+0000","last_clean":"2024-09-14T04:25:39.638882+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:25:39.638882+0000","last_undegraded":"2024-09-14T04:25:39.638882+0000","last_fullsized":"2024-09-14T04:25:39.638882+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":896,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.633211+0000","last_change":"2024-09-14T04:25:10.214026+0000","last_active":"2024-09-14T04:25:39.633211+0000","last_peered":"2024-09-14T04:25:39.633211+0000","last_clean":"2024-09-14T04:25:39.633211+0000","last_became_active":"2024-09-14T04:25:10.212511+0000","last_became_peered":"2024-09-14T04:25:10.212511+0000","last_unstale":"2024-09-14T04:25:39.633211+0000","last_undegraded":"2024-09-14T04:25:39.633211+0000","last_fullsized":"2024-09-14T04:25:39.633211+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:20.633170+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":309,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605727+0000","last_change":"2024-09-14T04:25:35.559702+0000","last_active":"2024-09-14T04:25:38.605727+0000","last_peered":"2024-09-14T04:25:38.605727+0000","last_clean":"2024-09-14T04:25:38.605727+0000","last_became_active":"2024-09-14T04:25:35.559395+0000","last_became_peered":"2024-09-14T04:25:35.559395+0000","last_unstale":"2024-09-14T04:25:38.605727+0000","last_undegraded":"2024-09-14T04:25:38.605727+0000","last_fullsized":"2024-09-14T04:25:38.605727+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:51.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605720+0000","last_change":"2024-09-14T04:24:09.122059+0000","last_active":"2024-09-14T04:25:38.605720+0000","last_peered":"2024-09-14T04:25:38.605720+0000","last_clean":"2024-09-14T04:25:38.605720+0000","last_became_active":"2024-09-14T04:24:09.121824+0000","last_became_peered":"2024-09-14T04:24:09.121824+0000","last_unstale":"2024-09-14T04:25:38.605720+0000","last_undegraded":"2024-09-14T04:25:38.605720+0000","last_fullsized":"2024-09-14T04:25:38.605720+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:36.179815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":447,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.649075+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:25:43.649075+0000","last_peered":"2024-09-14T04:25:43.649075+0000","last_clean":"2024-09-14T04:25:43.649075+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:25:43.649075+0000","last_undegraded":"2024-09-14T04:25:43.649075+0000","last_fullsized":"2024-09-14T04:25:43.649075+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":549,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631568+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:25:39.631568+0000","last_peered":"2024-09-14T04:25:39.631568+0000","last_clean":"2024-09-14T04:25:39.631568+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:25:39.631568+0000","last_undegraded":"2024-09-14T04:25:39.631568+0000","last_fullsized":"2024-09-14T04:25:39.631568+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.607435+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:25:38.607435+0000","last_peered":"2024-09-14T04:25:38.607435+0000","last_clean":"2024-09-14T04:25:38.607435+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:25:38.607435+0000","last_undegraded":"2024-09-14T04:25:38.607435+0000","last_fullsized":"2024-09-14T04:25:38.607435+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":455,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640918+0000","last_change":"2024-09-14T04:23:27.569599+0000","last_active":"2024-09-14T04:25:43.640918+0000","last_peered":"2024-09-14T04:25:43.640918+0000","last_clean":"2024-09-14T04:25:43.640918+0000","last_became_active":"2024-09-14T04:23:27.569318+0000","last_became_peered":"2024-09-14T04:23:27.569318+0000","last_unstale":"2024-09-14T04:25:43.640918+0000","last_undegraded":"2024-09-14T04:25:43.640918+0000","last_fullsized":"2024-09-14T04:25:43.640918+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:26.941017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":463,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.643637+0000","last_change":"2024-09-14T04:25:35.564324+0000","last_active":"2024-09-14T04:25:43.643637+0000","last_peered":"2024-09-14T04:25:43.643637+0000","last_clean":"2024-09-14T04:25:43.643637+0000","last_became_active":"2024-09-14T04:25:35.562918+0000","last_became_peered":"2024-09-14T04:25:35.562918+0000","last_unstale":"2024-09-14T04:25:43.643637+0000","last_undegraded":"2024-09-14T04:25:43.643637+0000","last_fullsized":"2024-09-14T04:25:43.643637+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:16.172185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":709,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.628865+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:25:39.628865+0000","last_peered":"2024-09-14T04:25:39.628865+0000","last_clean":"2024-09-14T04:25:39.628865+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:25:39.628865+0000","last_undegraded":"2024-09-14T04:25:39.628865+0000","last_fullsized":"2024-09-14T04:25:39.628865+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":267,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605721+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:25:38.605721+0000","last_peered":"2024-09-14T04:25:38.605721+0000","last_clean":"2024-09-14T04:25:38.605721+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:25:38.605721+0000","last_undegraded":"2024-09-14T04:25:38.605721+0000","last_fullsized":"2024-09-14T04:25:38.605721+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":601,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.640231+0000","last_change":"2024-09-14T04:19:47.450267+0000","last_active":"2024-09-14T04:25:43.640231+0000","last_peered":"2024-09-14T04:25:43.640231+0000","last_clean":"2024-09-14T04:25:43.640231+0000","last_became_active":"2024-09-14T04:19:47.449228+0000","last_became_peered":"2024-09-14T04:19:47.449228+0000","last_unstale":"2024-09-14T04:25:43.640231+0000","last_undegraded":"2024-09-14T04:25:43.640231+0000","last_fullsized":"2024-09-14T04:25:43.640231+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:05:44.565360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":120,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.633494+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:25:39.633494+0000","last_peered":"2024-09-14T04:25:39.633494+0000","last_clean":"2024-09-14T04:25:39.633494+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:25:39.633494+0000","last_undegraded":"2024-09-14T04:25:39.633494+0000","last_fullsized":"2024-09-14T04:25:39.633494+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":461,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.645852+0000","last_change":"2024-09-14T04:23:27.573928+0000","last_active":"2024-09-14T04:25:43.645852+0000","last_peered":"2024-09-14T04:25:43.645852+0000","last_clean":"2024-09-14T04:25:43.645852+0000","last_became_active":"2024-09-14T04:23:27.573557+0000","last_became_peered":"2024-09-14T04:23:27.573557+0000","last_unstale":"2024-09-14T04:25:43.645852+0000","last_undegraded":"2024-09-14T04:25:43.645852+0000","last_fullsized":"2024-09-14T04:25:43.645852+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:07.509590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610539+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:25:38.610539+0000","last_peered":"2024-09-14T04:25:38.610539+0000","last_clean":"2024-09-14T04:25:38.610539+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:25:38.610539+0000","last_undegraded":"2024-09-14T04:25:38.610539+0000","last_fullsized":"2024-09-14T04:25:38.610539+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":365,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605913+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:25:38.605913+0000","last_peered":"2024-09-14T04:25:38.605913+0000","last_clean":"2024-09-14T04:25:38.605913+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:25:38.605913+0000","last_undegraded":"2024-09-14T04:25:38.605913+0000","last_fullsized":"2024-09-14T04:25:38.605913+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610404+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:25:38.610404+0000","last_peered":"2024-09-14T04:25:38.610404+0000","last_clean":"2024-09-14T04:25:38.610404+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:25:38.610404+0000","last_undegraded":"2024-09-14T04:25:38.610404+0000","last_fullsized":"2024-09-14T04:25:38.610404+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":361,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.605598+0000","last_change":"2024-09-14T04:24:07.980704+0000","last_active":"2024-09-14T04:25:38.605598+0000","last_peered":"2024-09-14T04:25:38.605598+0000","last_clean":"2024-09-14T04:25:38.605598+0000","last_became_active":"2024-09-14T04:24:07.980309+0000","last_became_peered":"2024-09-14T04:24:07.980309+0000","last_unstale":"2024-09-14T04:25:38.605598+0000","last_undegraded":"2024-09-14T04:25:38.605598+0000","last_fullsized":"2024-09-14T04:25:38.605598+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:36.154454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":598,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631113+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:25:39.631113+0000","last_peered":"2024-09-14T04:25:39.631113+0000","last_clean":"2024-09-14T04:25:39.631113+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:25:39.631113+0000","last_undegraded":"2024-09-14T04:25:39.631113+0000","last_fullsized":"2024-09-14T04:25:39.631113+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":269,"reported_epoch":418,"state":"active+clean","last_fresh":"2024-09-14T04:25:39.631458+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:25:39.631458+0000","last_peered":"2024-09-14T04:25:39.631458+0000","last_clean":"2024-09-14T04:25:39.631458+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:25:39.631458+0000","last_undegraded":"2024-09-14T04:25:39.631458+0000","last_fullsized":"2024-09-14T04:25:39.631458+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":399,"reported_epoch":417,"state":"active+clean","last_fresh":"2024-09-14T04:25:38.610010+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:25:38.610010+0000","last_peered":"2024-09-14T04:25:38.610010+0000","last_clean":"2024-09-14T04:25:38.610010+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:25:38.610010+0000","last_undegraded":"2024-09-14T04:25:38.610010+0000","last_fullsized":"2024-09-14T04:25:38.610010+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":495,"reported_epoch":419,"state":"active+clean","last_fresh":"2024-09-14T04:25:43.649189+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:25:43.649189+0000","last_peered":"2024-09-14T04:25:43.649189+0000","last_clean":"2024-09-14T04:25:43.649189+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:25:43.649189+0000","last_undegraded":"2024-09-14T04:25:43.649189+0000","last_fullsized":"2024-09-14T04:25:43.649189+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3634954240,"num_objects":55465,"num_object_clones":0,"num_object_copies":166395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3569,"num_objects_unfound":0,"num_objects_dirty":55465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":55465,"num_write_kb":3549760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1805,"num_bytes_recovered":118292480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11072241664,"data_stored":11072241664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55465,"ondisk_log_size":55465,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":12322,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2506969,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201821,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463459,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834132,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527236,"kb_used_data":2369396,"kb_used_omap":5,"kb_used_meta":157818,"kb_avail":91217916,"statfs":{"total":95995035648,"available":93407145984,"internally_reserved":0,"allocated":2426261504,"data_stored":2423550880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":161605693},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986731,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3410880,"kb_used_data":2102584,"kb_used_omap":1,"kb_used_meta":1308286,"kb_avail":90334272,"statfs":{"total":95995035648,"available":92502294528,"internally_reserved":0,"allocated":2153046016,"data_stored":2150327171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1339685318},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748760,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2232828,"kb_used_data":1297012,"kb_used_omap":4,"kb_used_meta":935803,"kb_avail":91512324,"statfs":{"total":95995035648,"available":93708619776,"internally_reserved":0,"allocated":1328140288,"data_stored":1325429446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":958262828},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286981,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2421384,"kb_used_data":2000704,"kb_used_omap":10,"kb_used_meta":420661,"kb_avail":91323768,"statfs":{"total":95995035648,"available":93515538432,"internally_reserved":0,"allocated":2048720896,"data_stored":2046002978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":430757463},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468303,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2276856,"kb_used_data":1316140,"kb_used_omap":5,"kb_used_meta":960698,"kb_avail":91468296,"statfs":{"total":95995035648,"available":93663535104,"internally_reserved":0,"allocated":1347727360,"data_stored":1345022627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":983755103},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342419,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226592,"kb_used_data":1617748,"kb_used_omap":10,"kb_used_meta":608821,"kb_avail":91518560,"statfs":{"total":95995035648,"available":93715005440,"internally_reserved":0,"allocated":1656573952,"data_stored":1653843575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":623432914},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1650524160,"data_stored":1650524160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1356398592,"data_stored":1356398592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2071199744,"data_stored":2071199744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1335427072,"data_stored":1335427072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2204041216,"data_stored":2204041216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2454650880,"data_stored":2454650880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:25:46.076 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:25:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:45 smithi012 ceph-mon[101180]: osdmap e420: 8 total, 8 up, 6 in 2024-09-14T04:25:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:45 smithi012 ceph-mon[101180]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:25:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:45 smithi012 ceph-mon[101180]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 2951/170544 objects misplaced (1.730%); 18 MiB/s, 289 objects/s recovering 2024-09-14T04:25:46.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:46.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:46.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:46.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:46.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:46.371 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:46.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:46.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:46.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:46.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:46.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:46.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 61644 61628 104.079 83.4375 0.00646139 0.00960233 2024-09-14T04:25:46.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:46.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:46.936 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:46] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.51.0" 2024-09-14T04:25:46.973 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:46.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:47.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.109 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:47.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:47.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:47.242 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:47.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:47.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:47.383 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:47.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:47.528 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:47.630 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:47.666 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:47.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:47.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 62925 62909 103.447 80.0625 0.00458989 0.00964066 2024-09-14T04:25:47.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:47.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:47.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:47.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:47.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:48.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:48.065 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:48.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:48.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:48.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:48.325 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:48.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:48.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:48.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:48.591 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:48.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:48.736 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:48.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 16 64227 64211 102.881 81.375 0.0135709 0.00971434 2024-09-14T04:25:48.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:48.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:48 smithi012 ceph-mon[101180]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 2643/182013 objects misplaced (1.452%); 16 MiB/s, 259 objects/s recovering 2024-09-14T04:25:48.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:48.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:48 smithi001 ceph-mon[107399]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 2643/182013 objects misplaced (1.452%); 16 MiB/s, 259 objects/s recovering 2024-09-14T04:25:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:48 smithi001 ceph-mon[112165]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 2643/182013 objects misplaced (1.452%); 16 MiB/s, 259 objects/s recovering 2024-09-14T04:25:48.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:49.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:49.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:49.154 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:49.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:49.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:49.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:49.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:49.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:49.552 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:49.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:49.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:49.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:49.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:25:49.789219+0000 min lat: 0.00114104 max lat: 2.49143 avg lat: 0.00976027 2024-09-14T04:25:49.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:25:49.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 65566 65550 102.401 83.6875 0.00607409 0.00976027 2024-09-14T04:25:49.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:49.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:49.967 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:49.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:49.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:50.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:50.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:50.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:50.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:50.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:50.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:50.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:50.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:50.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:50.664 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:50.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:50.789 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 66902 66886 101.939 83.5 0.0315095 0.00980516 2024-09-14T04:25:50.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:50 smithi012 ceph-mon[101180]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 2643/183225 objects misplaced (1.442%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:25:50.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:50.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:50 smithi001 ceph-mon[112165]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 2643/183225 objects misplaced (1.442%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:25:50.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:50.875 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:50 smithi001 ceph-mon[107399]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 2643/183225 objects misplaced (1.442%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:25:50.876 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:50.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:50.934 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:50.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:51.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:51.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:51.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:51.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:51.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:51.344 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:51.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:51.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:51.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:51.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:51.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:51.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:51.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 15 68119 68104 101.324 76.125 0.0168338 0.00986424 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]} : dispatch 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]} : dispatch 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"} : dispatch 2024-09-14T04:25:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"} : dispatch 2024-09-14T04:25:51.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:51.911 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:51.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:51.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]} : dispatch 2024-09-14T04:25:51.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"} : dispatch 2024-09-14T04:25:51.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"} : dispatch 2024-09-14T04:25:52.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:52.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:52.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:52.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:52.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:52.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:52.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:52.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:52.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:52.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:52.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:52.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:52.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 2381/195648 objects misplaced (1.217%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"}]': finished 2024-09-14T04:25:52.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[107399]: osdmap e421: 8 total, 8 up, 6 in 2024-09-14T04:25:52.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 2381/195648 objects misplaced (1.217%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:25:52.723 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"}]': finished 2024-09-14T04:25:52.724 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:52 smithi001 ceph-mon[112165]: osdmap e421: 8 total, 8 up, 6 in 2024-09-14T04:25:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:52.790 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 69365 69349 100.777 77.8125 0.0234206 0.00991698 2024-09-14T04:25:52.825 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 2381/195648 objects misplaced (1.217%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [1, 5]}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [0, 5]}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1a"}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.1d"}]': finished 2024-09-14T04:25:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:52 smithi012 ceph-mon[101180]: osdmap e421: 8 total, 8 up, 6 in 2024-09-14T04:25:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.867 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:52.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:52.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:52.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:52.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:52.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:53.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.140 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:53.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:53.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:53.278 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:53.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:53.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:53.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:53.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:53.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:53.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:53 smithi001 ceph-mon[112165]: osdmap e422: 8 total, 8 up, 6 in 2024-09-14T04:25:53.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:53 smithi001 ceph-mon[107399]: osdmap e422: 8 total, 8 up, 6 in 2024-09-14T04:25:53.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:53.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 70612 70596 100.251 77.9375 0.00419702 0.00996894 2024-09-14T04:25:53.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:53.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:53 smithi012 ceph-mon[101180]: osdmap e422: 8 total, 8 up, 6 in 2024-09-14T04:25:53.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:53.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:53.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:53.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:54.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:54.136 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:54.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:54.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:54.325 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:54.336 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:25:54.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:54.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:54.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:54.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:54.587 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:54.596 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:54 smithi001 ceph-mon[107399]: pgmap v1392: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 2381/201630 objects misplaced (1.181%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:25:54.596 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:54 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:25:54.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.735 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:54.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:54.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 71848 71832 99.7399 77.25 0.00987179 0.0100211 2024-09-14T04:25:54.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:54 smithi012 ceph-mon[101180]: pgmap v1392: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 2381/201630 objects misplaced (1.181%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:25:54.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:54 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:25:54.863 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:54 smithi001 ceph-mon[112165]: pgmap v1392: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 2381/201630 objects misplaced (1.181%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:25:54.863 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:54 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:25:54.868 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:54.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:54.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:55.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:55.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:55.259 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:55.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:55.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:55.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:55.614 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:25:55.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:55.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:55.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 72968 72952 99.0933 70 0.0104208 0.010086 2024-09-14T04:25:55.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:55.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:55.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:55.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:55.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/703166561' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:55.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:55.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/703166561' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:55.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:56.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:56.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/703166561' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:25:56.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:25:56.140 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:56.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:25:56.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:56.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:56.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:56.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:56.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:56.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:56.555 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:56.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:56.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:56.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[107399]: pgmap v1393: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 2381/207567 objects misplaced (1.147%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:25:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:25:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:56.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[107399]: osdmap e423: 8 total, 8 up, 6 in 2024-09-14T04:25:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[112165]: pgmap v1393: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 2381/207567 objects misplaced (1.147%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:25:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:25:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:56.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-mon[112165]: osdmap e423: 8 total, 8 up, 6 in 2024-09-14T04:25:56.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:56.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 74255 74239 98.6963 80.4375 0.00861748 0.0101269 2024-09-14T04:25:56.820 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:56.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:56.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:56.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:25:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:25:56] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-14T04:25:57.052 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:57.092 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:57.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:56 smithi012 ceph-mon[101180]: pgmap v1393: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 107 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 2381/207567 objects misplaced (1.147%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:25:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:56 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:25:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:56 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:25:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:56 smithi012 ceph-mon[101180]: osdmap e423: 8 total, 8 up, 6 in 2024-09-14T04:25:57.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:57.230 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:57.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:57.383 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:57.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:57.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:57.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:57.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:57.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:57.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:57.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 75490 75474 98.2481 77.1875 0.0101638 0.0101724 2024-09-14T04:25:57.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.810 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:57.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:57.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:57.943 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[107399]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 2107/220212 objects misplaced (0.957%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:25:57.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:25:57.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:25:57.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[112165]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 2107/220212 objects misplaced (0.957%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:25:57.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:25:57.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:57 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:25:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:57.945 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:57.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:58.021 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:25:58.021 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:25:58.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:25:58.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:25:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:57 smithi012 ceph-mon[101180]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 2107/220212 objects misplaced (0.957%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:25:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:57 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:25:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:57 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:25:58.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:25:58.222 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:25:58.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:58.353 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:58.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:58.455 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:58.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:58.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:58.640 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:58.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.770 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:58.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:58.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 76514 76498 97.5492 64 0.00626776 0.0102454 2024-09-14T04:25:58.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:58.893 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:58.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:58.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1467648451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:58.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1467648451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:58.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:59.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:59.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1467648451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:25:59.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:25:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.157 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:59.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:25:59.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:59.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:25:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:59.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:59.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:25:59.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:25:59.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:25:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.588 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:59.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:25:59.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:25:59.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:25:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:25:59.793 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 77479 77463 96.8045 60.3125 0.0155561 0.0103249 2024-09-14T04:25:59.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:25:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:25:59.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:25:59.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:25:59.961 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:59 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:59.961 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:25:59 smithi012 ceph-mon[101180]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 2107/220212 objects misplaced (0.957%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:25:59.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:59 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:59.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:25:59 smithi001 ceph-mon[112165]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 2107/220212 objects misplaced (0.957%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:25:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:59 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:25:59 smithi001 ceph-mon[107399]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 2107/220212 objects misplaced (0.957%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:25:59.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.012 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:00.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:00.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:00.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:00.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:00.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:00.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:00.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:00.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:00.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:00.587 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:00.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:00.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.722 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:00.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:00.794 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 78735 78719 96.445 78.5 0.0117689 0.0103629 2024-09-14T04:26:00.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:00.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:00.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:00.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:00.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:01.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:01.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:01.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:01.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:01.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:01.320 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:26:01.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:01.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:26:01.469 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:26:01.470 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:26:01.568 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:01.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:01.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:01.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:01.795 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 79871 79855 95.9534 71 0.0108427 0.0104158 2024-09-14T04:26:01.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:01.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:01.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:02.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:02.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:02.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:02.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:02.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:02.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:02.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:02.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:02.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:02.581 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:02.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:02.678 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:02 smithi001 ceph-mon[107399]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 1840/231888 objects misplaced (0.793%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:02.678 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:02 smithi001 ceph-mon[112165]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 1840/231888 objects misplaced (0.793%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:02.690 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:02.736 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:02.777 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:02.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 80988 80972 95.459 69.8125 0.0169972 0.0104685 2024-09-14T04:26:02.810 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:02 smithi012 ceph-mon[101180]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 1840/231888 objects misplaced (0.793%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:02.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:02.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:02.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:03.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:03.113 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:26:03.117 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:26:03.146 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:03.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:03.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:03.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:03.382 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:03.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:03.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:03.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:03.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:03.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:03.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 82081 82065 94.9563 68.3125 0.0307946 0.0105253 2024-09-14T04:26:03.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:03.862 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:03.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:03.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:03.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:03.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:04.059 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1397,"stamp":"2024-09-14T04:26:01.500575+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5064589496,"num_objects":77296,"num_object_clones":0,"num_object_copies":231888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1840,"num_objects_unfound":0,"num_objects_dirty":77296,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":77344,"num_write_kb":4945423,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3601,"num_bytes_recovered":234123515,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80458,"ondisk_log_size":80458,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19460800,"kb_used_data":14929740,"kb_used_omap":38,"kb_used_meta":4530969,"kb_avail":543010112,"statfs":{"total":575970213888,"available":556042354688,"internally_reserved":0,"allocated":15288053760,"data_stored":15271576113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39753,"internal_metadata":4639712439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,0,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":660996096,"num_objects":10086,"num_object_clones":0,"num_object_copies":30258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-541,"num_objects_unfound":0,"num_objects_dirty":10086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10086,"num_write_kb":645504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35454976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.928890"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":649,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620767+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:25:55.620767+0000","last_peered":"2024-09-14T04:25:55.620767+0000","last_clean":"2024-09-14T04:25:55.620767+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:25:55.620767+0000","last_undegraded":"2024-09-14T04:25:55.620767+0000","last_fullsized":"2024-09-14T04:25:55.620767+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623264+0000","last_change":"2024-09-14T04:23:28.577777+0000","last_active":"2024-09-14T04:25:55.623264+0000","last_peered":"2024-09-14T04:25:55.623264+0000","last_clean":"2024-09-14T04:25:55.623264+0000","last_became_active":"2024-09-14T04:23:28.577155+0000","last_became_peered":"2024-09-14T04:23:28.577155+0000","last_unstale":"2024-09-14T04:25:55.623264+0000","last_undegraded":"2024-09-14T04:25:55.623264+0000","last_fullsized":"2024-09-14T04:25:55.623264+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:01.212541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":171,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575714+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:25:52.575714+0000","last_peered":"2024-09-14T04:25:52.575714+0000","last_clean":"2024-09-14T04:25:52.575714+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:25:52.575714+0000","last_undegraded":"2024-09-14T04:25:52.575714+0000","last_fullsized":"2024-09-14T04:25:52.575714+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":761,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570443+0000","last_change":"2024-09-14T04:24:49.569053+0000","last_active":"2024-09-14T04:25:52.570443+0000","last_peered":"2024-09-14T04:25:52.570443+0000","last_clean":"2024-09-14T04:25:52.570443+0000","last_became_active":"2024-09-14T04:24:49.568690+0000","last_became_peered":"2024-09-14T04:24:49.568690+0000","last_unstale":"2024-09-14T04:25:52.570443+0000","last_undegraded":"2024-09-14T04:25:52.570443+0000","last_fullsized":"2024-09-14T04:25:52.570443+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:19:37.952706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":26,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569752+0000","last_change":"2024-09-14T04:25:25.582150+0000","last_active":"2024-09-14T04:25:52.569752+0000","last_peered":"2024-09-14T04:25:52.569752+0000","last_clean":"2024-09-14T04:25:52.569752+0000","last_became_active":"2024-09-14T04:25:25.581683+0000","last_became_peered":"2024-09-14T04:25:25.581683+0000","last_unstale":"2024-09-14T04:25:52.569752+0000","last_undegraded":"2024-09-14T04:25:52.569752+0000","last_fullsized":"2024-09-14T04:25:52.569752+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:10.188214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":465,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620994+0000","last_change":"2024-09-14T04:25:35.562683+0000","last_active":"2024-09-14T04:25:55.620994+0000","last_peered":"2024-09-14T04:25:55.620994+0000","last_clean":"2024-09-14T04:25:55.620994+0000","last_became_active":"2024-09-14T04:25:35.562281+0000","last_became_peered":"2024-09-14T04:25:35.562281+0000","last_unstale":"2024-09-14T04:25:55.620994+0000","last_undegraded":"2024-09-14T04:25:55.620994+0000","last_fullsized":"2024-09-14T04:25:55.620994+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:09.493757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":938,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624692+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:25:55.624692+0000","last_peered":"2024-09-14T04:25:55.624692+0000","last_clean":"2024-09-14T04:25:55.624692+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:25:55.624692+0000","last_undegraded":"2024-09-14T04:25:55.624692+0000","last_fullsized":"2024-09-14T04:25:55.624692+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":451,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624255+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:25:55.624255+0000","last_peered":"2024-09-14T04:25:55.624255+0000","last_clean":"2024-09-14T04:25:55.624255+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:25:55.624255+0000","last_undegraded":"2024-09-14T04:25:55.624255+0000","last_fullsized":"2024-09-14T04:25:55.624255+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":377,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575280+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:25:52.575280+0000","last_peered":"2024-09-14T04:25:52.575280+0000","last_clean":"2024-09-14T04:25:52.575280+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:25:52.575280+0000","last_undegraded":"2024-09-14T04:25:52.575280+0000","last_fullsized":"2024-09-14T04:25:52.575280+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":589,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.573784+0000","last_change":"2024-09-14T04:24:49.572838+0000","last_active":"2024-09-14T04:25:52.573784+0000","last_peered":"2024-09-14T04:25:52.573784+0000","last_clean":"2024-09-14T04:25:52.573784+0000","last_became_active":"2024-09-14T04:24:49.572388+0000","last_became_peered":"2024-09-14T04:24:49.572388+0000","last_unstale":"2024-09-14T04:25:52.573784+0000","last_undegraded":"2024-09-14T04:25:52.573784+0000","last_fullsized":"2024-09-14T04:25:52.573784+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:22.578056+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615918+0000","last_change":"2024-09-14T04:25:35.565538+0000","last_active":"2024-09-14T04:25:55.615918+0000","last_peered":"2024-09-14T04:25:55.615918+0000","last_clean":"2024-09-14T04:25:55.615918+0000","last_became_active":"2024-09-14T04:25:35.565210+0000","last_became_peered":"2024-09-14T04:25:35.565210+0000","last_unstale":"2024-09-14T04:25:55.615918+0000","last_undegraded":"2024-09-14T04:25:55.615918+0000","last_fullsized":"2024-09-14T04:25:55.615918+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:08.538693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":532,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621620+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:25:55.621620+0000","last_peered":"2024-09-14T04:25:55.621620+0000","last_clean":"2024-09-14T04:25:55.621620+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:25:55.621620+0000","last_undegraded":"2024-09-14T04:25:55.621620+0000","last_fullsized":"2024-09-14T04:25:55.621620+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"423'4570","reported_seq":4631,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:57.166342+0000","last_change":"2024-09-14T04:25:02.146250+0000","last_active":"2024-09-14T04:25:57.166342+0000","last_peered":"2024-09-14T04:25:57.166342+0000","last_clean":"2024-09-14T04:25:57.166342+0000","last_became_active":"2024-09-14T04:25:02.145730+0000","last_became_peered":"2024-09-14T04:25:02.145730+0000","last_unstale":"2024-09-14T04:25:57.166342+0000","last_undegraded":"2024-09-14T04:25:57.166342+0000","last_fullsized":"2024-09-14T04:25:57.166342+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:04.224581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299499520,"num_objects":4570,"num_object_clones":0,"num_object_copies":13710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4570,"num_write_kb":292480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":744,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.577236+0000","last_change":"2024-09-14T04:24:49.572850+0000","last_active":"2024-09-14T04:25:52.577236+0000","last_peered":"2024-09-14T04:25:52.577236+0000","last_clean":"2024-09-14T04:25:52.577236+0000","last_became_active":"2024-09-14T04:24:49.572405+0000","last_became_peered":"2024-09-14T04:24:49.572405+0000","last_unstale":"2024-09-14T04:25:52.577236+0000","last_undegraded":"2024-09-14T04:25:52.577236+0000","last_fullsized":"2024-09-14T04:25:52.577236+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:06.709118+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":139,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575966+0000","last_change":"2024-09-14T04:23:28.575634+0000","last_active":"2024-09-14T04:25:52.575966+0000","last_peered":"2024-09-14T04:25:52.575966+0000","last_clean":"2024-09-14T04:25:52.575966+0000","last_became_active":"2024-09-14T04:23:28.558391+0000","last_became_peered":"2024-09-14T04:23:28.558391+0000","last_unstale":"2024-09-14T04:25:52.575966+0000","last_undegraded":"2024-09-14T04:25:52.575966+0000","last_fullsized":"2024-09-14T04:25:52.575966+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:03.845925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.596308+0000","last_change":"2024-09-14T04:25:52.595074+0000","last_active":"2024-09-14T04:25:52.596308+0000","last_peered":"2024-09-14T04:25:52.596308+0000","last_clean":"2024-09-14T04:25:52.596308+0000","last_became_active":"2024-09-14T04:25:52.594744+0000","last_became_peered":"2024-09-14T04:25:52.594744+0000","last_unstale":"2024-09-14T04:25:52.596308+0000","last_undegraded":"2024-09-14T04:25:52.596308+0000","last_fullsized":"2024-09-14T04:25:52.596308+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:10.267321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"423'4935","reported_seq":4996,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:01.078383+0000","last_change":"2024-09-14T04:25:01.992464+0000","last_active":"2024-09-14T04:26:01.078383+0000","last_peered":"2024-09-14T04:26:01.078383+0000","last_clean":"2024-09-14T04:26:01.078383+0000","last_became_active":"2024-09-14T04:25:01.992117+0000","last_became_peered":"2024-09-14T04:25:01.992117+0000","last_unstale":"2024-09-14T04:26:01.078383+0000","last_undegraded":"2024-09-14T04:26:01.078383+0000","last_fullsized":"2024-09-14T04:26:01.078383+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:21.289671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4935,"num_write_kb":315840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":991,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615687+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:25:55.615687+0000","last_peered":"2024-09-14T04:25:55.615687+0000","last_clean":"2024-09-14T04:25:55.615687+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:25:55.615687+0000","last_undegraded":"2024-09-14T04:25:55.615687+0000","last_fullsized":"2024-09-14T04:25:55.615687+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":36,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575415+0000","last_change":"2024-09-14T04:25:07.963635+0000","last_active":"2024-09-14T04:25:52.575415+0000","last_peered":"2024-09-14T04:25:52.575415+0000","last_clean":"2024-09-14T04:25:52.575415+0000","last_became_active":"2024-09-14T04:25:07.959540+0000","last_became_peered":"2024-09-14T04:25:07.959540+0000","last_unstale":"2024-09-14T04:25:52.575415+0000","last_undegraded":"2024-09-14T04:25:52.575415+0000","last_fullsized":"2024-09-14T04:25:52.575415+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:05.353685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":448,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575101+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:25:52.575101+0000","last_peered":"2024-09-14T04:25:52.575101+0000","last_clean":"2024-09-14T04:25:52.575101+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:25:52.575101+0000","last_undegraded":"2024-09-14T04:25:52.575101+0000","last_fullsized":"2024-09-14T04:25:52.575101+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"423'4837","reported_seq":4898,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:59.480123+0000","last_change":"2024-09-14T04:25:02.146419+0000","last_active":"2024-09-14T04:25:59.480123+0000","last_peered":"2024-09-14T04:25:59.480123+0000","last_clean":"2024-09-14T04:25:59.480123+0000","last_became_active":"2024-09-14T04:25:02.145877+0000","last_became_peered":"2024-09-14T04:25:02.145877+0000","last_unstale":"2024-09-14T04:25:59.480123+0000","last_undegraded":"2024-09-14T04:25:59.480123+0000","last_fullsized":"2024-09-14T04:25:59.480123+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4837,"log_dups_size":0,"ondisk_log_size":4837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:31.774348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4837,"num_write_kb":309568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":444,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570413+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:25:52.570413+0000","last_peered":"2024-09-14T04:25:52.570413+0000","last_clean":"2024-09-14T04:25:52.570413+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:25:52.570413+0000","last_undegraded":"2024-09-14T04:25:52.570413+0000","last_fullsized":"2024-09-14T04:25:52.570413+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":30,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569680+0000","last_change":"2024-09-14T04:25:25.584570+0000","last_active":"2024-09-14T04:25:52.569680+0000","last_peered":"2024-09-14T04:25:52.569680+0000","last_clean":"2024-09-14T04:25:52.569680+0000","last_became_active":"2024-09-14T04:25:25.581721+0000","last_became_peered":"2024-09-14T04:25:25.581721+0000","last_unstale":"2024-09-14T04:25:52.569680+0000","last_undegraded":"2024-09-14T04:25:52.569680+0000","last_fullsized":"2024-09-14T04:25:52.569680+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:59.618866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":438,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623950+0000","last_change":"2024-09-14T04:24:08.985773+0000","last_active":"2024-09-14T04:25:55.623950+0000","last_peered":"2024-09-14T04:25:55.623950+0000","last_clean":"2024-09-14T04:25:55.623950+0000","last_became_active":"2024-09-14T04:24:08.985458+0000","last_became_peered":"2024-09-14T04:24:08.985458+0000","last_unstale":"2024-09-14T04:25:55.623950+0000","last_undegraded":"2024-09-14T04:25:55.623950+0000","last_fullsized":"2024-09-14T04:25:55.623950+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:10.470781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"423'4983","reported_seq":7546,"reported_epoch":423,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:26:01.027228+0000","last_change":"2024-09-14T04:25:52.621612+0000","last_active":"2024-09-14T04:26:01.027228+0000","last_peered":"2024-09-14T04:26:01.027228+0000","last_clean":"2024-09-14T04:25:34.547607+0000","last_became_active":"2024-09-14T04:25:37.041836+0000","last_became_peered":"2024-09-14T04:25:37.041836+0000","last_unstale":"2024-09-14T04:26:01.027228+0000","last_undegraded":"2024-09-14T04:26:01.027228+0000","last_fullsized":"2024-09-14T04:25:36.555651+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4983,"log_dups_size":0,"ondisk_log_size":4983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.438646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326565888,"num_objects":4983,"num_object_clones":0,"num_object_copies":14949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1840,"num_objects_unfound":0,"num_objects_dirty":4983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4983,"num_write_kb":318912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1254,"num_bytes_recovered":82182144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1840},{"shards":"4,5","objects":3143}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":399,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575427+0000","last_change":"2024-09-14T04:25:10.210264+0000","last_active":"2024-09-14T04:25:52.575427+0000","last_peered":"2024-09-14T04:25:52.575427+0000","last_clean":"2024-09-14T04:25:52.575427+0000","last_became_active":"2024-09-14T04:25:10.209992+0000","last_became_peered":"2024-09-14T04:25:10.209992+0000","last_unstale":"2024-09-14T04:25:52.575427+0000","last_undegraded":"2024-09-14T04:25:52.575427+0000","last_fullsized":"2024-09-14T04:25:52.575427+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:50.222369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:50.222369+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":371,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.571238+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:25:52.571238+0000","last_peered":"2024-09-14T04:25:52.571238+0000","last_clean":"2024-09-14T04:25:52.571238+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:25:52.571238+0000","last_undegraded":"2024-09-14T04:25:52.571238+0000","last_fullsized":"2024-09-14T04:25:52.571238+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":444,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621927+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:25:55.621927+0000","last_peered":"2024-09-14T04:25:55.621927+0000","last_clean":"2024-09-14T04:25:55.621927+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:25:55.621927+0000","last_undegraded":"2024-09-14T04:25:55.621927+0000","last_fullsized":"2024-09-14T04:25:55.621927+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"423'4953","reported_seq":4987,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:00.069175+0000","last_change":"2024-09-14T04:25:01.992668+0000","last_active":"2024-09-14T04:26:00.069175+0000","last_peered":"2024-09-14T04:26:00.069175+0000","last_clean":"2024-09-14T04:26:00.069175+0000","last_became_active":"2024-09-14T04:25:01.992359+0000","last_became_peered":"2024-09-14T04:25:01.992359+0000","last_unstale":"2024-09-14T04:26:00.069175+0000","last_undegraded":"2024-09-14T04:26:00.069175+0000","last_fullsized":"2024-09-14T04:26:00.069175+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:08.434277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324599808,"num_objects":4953,"num_object_clones":0,"num_object_copies":14859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4953,"num_write_kb":316992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":732,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.619623+0000","last_change":"2024-09-14T04:24:10.193742+0000","last_active":"2024-09-14T04:25:55.619623+0000","last_peered":"2024-09-14T04:25:55.619623+0000","last_clean":"2024-09-14T04:25:55.619623+0000","last_became_active":"2024-09-14T04:24:10.193277+0000","last_became_peered":"2024-09-14T04:24:10.193277+0000","last_unstale":"2024-09-14T04:25:55.619623+0000","last_undegraded":"2024-09-14T04:25:55.619623+0000","last_fullsized":"2024-09-14T04:25:55.619623+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:08.270894+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":448,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621560+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:25:55.621560+0000","last_peered":"2024-09-14T04:25:55.621560+0000","last_clean":"2024-09-14T04:25:55.621560+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:25:55.621560+0000","last_undegraded":"2024-09-14T04:25:55.621560+0000","last_fullsized":"2024-09-14T04:25:55.621560+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":424,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575316+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:25:52.575316+0000","last_peered":"2024-09-14T04:25:52.575316+0000","last_clean":"2024-09-14T04:25:52.575316+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:25:52.575316+0000","last_undegraded":"2024-09-14T04:25:52.575316+0000","last_fullsized":"2024-09-14T04:25:52.575316+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"423'4712","reported_seq":4773,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:57.187299+0000","last_change":"2024-09-14T04:25:01.995435+0000","last_active":"2024-09-14T04:25:57.187299+0000","last_peered":"2024-09-14T04:25:57.187299+0000","last_clean":"2024-09-14T04:25:57.187299+0000","last_became_active":"2024-09-14T04:25:01.995177+0000","last_became_peered":"2024-09-14T04:25:01.995177+0000","last_unstale":"2024-09-14T04:25:57.187299+0000","last_undegraded":"2024-09-14T04:25:57.187299+0000","last_fullsized":"2024-09-14T04:25:57.187299+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4712,"log_dups_size":0,"ondisk_log_size":4712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:02.315254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308805632,"num_objects":4712,"num_object_clones":0,"num_object_copies":14136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4712,"num_write_kb":301568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":657,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570221+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:25:52.570221+0000","last_peered":"2024-09-14T04:25:52.570221+0000","last_clean":"2024-09-14T04:25:52.570221+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:25:52.570221+0000","last_undegraded":"2024-09-14T04:25:52.570221+0000","last_fullsized":"2024-09-14T04:25:52.570221+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":526,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621144+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:25:55.621144+0000","last_peered":"2024-09-14T04:25:55.621144+0000","last_clean":"2024-09-14T04:25:55.621144+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:25:55.621144+0000","last_undegraded":"2024-09-14T04:25:55.621144+0000","last_fullsized":"2024-09-14T04:25:55.621144+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.576427+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:25:52.576427+0000","last_peered":"2024-09-14T04:25:52.576427+0000","last_clean":"2024-09-14T04:25:52.576427+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:25:52.576427+0000","last_undegraded":"2024-09-14T04:25:52.576427+0000","last_fullsized":"2024-09-14T04:25:52.576427+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"423'4941","reported_seq":9552,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:00.066099+0000","last_change":"2024-09-14T04:25:44.643768+0000","last_active":"2024-09-14T04:26:00.066099+0000","last_peered":"2024-09-14T04:26:00.066099+0000","last_clean":"2024-09-14T04:26:00.066099+0000","last_became_active":"2024-09-14T04:25:44.638986+0000","last_became_peered":"2024-09-14T04:25:44.638986+0000","last_unstale":"2024-09-14T04:26:00.066099+0000","last_undegraded":"2024-09-14T04:26:00.066099+0000","last_fullsized":"2024-09-14T04:26:00.066099+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4941,"log_dups_size":0,"ondisk_log_size":4941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:07.177695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323813376,"num_objects":4941,"num_object_clones":0,"num_object_copies":14823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4941,"num_write_kb":316224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":918,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624398+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:25:55.624398+0000","last_peered":"2024-09-14T04:25:55.624398+0000","last_clean":"2024-09-14T04:25:55.624398+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:25:55.624398+0000","last_undegraded":"2024-09-14T04:25:55.624398+0000","last_fullsized":"2024-09-14T04:25:55.624398+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":32,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575300+0000","last_change":"2024-09-14T04:25:07.967648+0000","last_active":"2024-09-14T04:25:52.575300+0000","last_peered":"2024-09-14T04:25:52.575300+0000","last_clean":"2024-09-14T04:25:52.575300+0000","last_became_active":"2024-09-14T04:25:07.967284+0000","last_became_peered":"2024-09-14T04:25:07.967284+0000","last_unstale":"2024-09-14T04:25:52.575300+0000","last_undegraded":"2024-09-14T04:25:52.575300+0000","last_fullsized":"2024-09-14T04:25:52.575300+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:29.967190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":430,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575854+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:25:52.575854+0000","last_peered":"2024-09-14T04:25:52.575854+0000","last_clean":"2024-09-14T04:25:52.575854+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:25:52.575854+0000","last_undegraded":"2024-09-14T04:25:52.575854+0000","last_fullsized":"2024-09-14T04:25:52.575854+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"423'4876","reported_seq":4910,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:00.065997+0000","last_change":"2024-09-14T04:25:02.146105+0000","last_active":"2024-09-14T04:26:00.065997+0000","last_peered":"2024-09-14T04:26:00.065997+0000","last_clean":"2024-09-14T04:26:00.065997+0000","last_became_active":"2024-09-14T04:25:02.145736+0000","last_became_peered":"2024-09-14T04:25:02.145736+0000","last_unstale":"2024-09-14T04:26:00.065997+0000","last_undegraded":"2024-09-14T04:26:00.065997+0000","last_fullsized":"2024-09-14T04:26:00.065997+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4876,"log_dups_size":0,"ondisk_log_size":4876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.907164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319553536,"num_objects":4876,"num_object_clones":0,"num_object_copies":14628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4876,"num_write_kb":312064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":627,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.573893+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:25:52.573893+0000","last_peered":"2024-09-14T04:25:52.573893+0000","last_clean":"2024-09-14T04:25:52.573893+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:25:52.573893+0000","last_undegraded":"2024-09-14T04:25:52.573893+0000","last_fullsized":"2024-09-14T04:25:52.573893+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615764+0000","last_change":"2024-09-14T04:25:35.561629+0000","last_active":"2024-09-14T04:25:55.615764+0000","last_peered":"2024-09-14T04:25:55.615764+0000","last_clean":"2024-09-14T04:25:55.615764+0000","last_became_active":"2024-09-14T04:25:35.561305+0000","last_became_peered":"2024-09-14T04:25:35.561305+0000","last_unstale":"2024-09-14T04:25:55.615764+0000","last_undegraded":"2024-09-14T04:25:55.615764+0000","last_fullsized":"2024-09-14T04:25:55.615764+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:39.574054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":688,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.625345+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:25:55.625345+0000","last_peered":"2024-09-14T04:25:55.625345+0000","last_clean":"2024-09-14T04:25:55.625345+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:25:55.625345+0000","last_undegraded":"2024-09-14T04:25:55.625345+0000","last_fullsized":"2024-09-14T04:25:55.625345+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"423'5071","reported_seq":5132,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:01.042061+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:26:01.042061+0000","last_peered":"2024-09-14T04:26:01.042061+0000","last_clean":"2024-09-14T04:26:01.042061+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:26:01.042061+0000","last_undegraded":"2024-09-14T04:26:01.042061+0000","last_fullsized":"2024-09-14T04:26:01.042061+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:09.181654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332333056,"num_objects":5071,"num_object_clones":0,"num_object_copies":15213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5071,"num_write_kb":324544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":787,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621762+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:25:55.621762+0000","last_peered":"2024-09-14T04:25:55.621762+0000","last_clean":"2024-09-14T04:25:55.621762+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:25:55.621762+0000","last_undegraded":"2024-09-14T04:25:55.621762+0000","last_fullsized":"2024-09-14T04:25:55.621762+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:17.530093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":361,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569571+0000","last_change":"2024-09-14T04:24:08.979706+0000","last_active":"2024-09-14T04:25:52.569571+0000","last_peered":"2024-09-14T04:25:52.569571+0000","last_clean":"2024-09-14T04:25:52.569571+0000","last_became_active":"2024-09-14T04:24:08.979397+0000","last_became_peered":"2024-09-14T04:24:08.979397+0000","last_unstale":"2024-09-14T04:25:52.569571+0000","last_undegraded":"2024-09-14T04:25:52.569571+0000","last_fullsized":"2024-09-14T04:25:52.569571+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:12.232467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.619119+0000","last_change":"2024-09-14T04:25:52.594667+0000","last_active":"2024-09-14T04:25:55.619119+0000","last_peered":"2024-09-14T04:25:55.619119+0000","last_clean":"2024-09-14T04:25:55.619119+0000","last_became_active":"2024-09-14T04:25:52.594400+0000","last_became_peered":"2024-09-14T04:25:52.594400+0000","last_unstale":"2024-09-14T04:25:55.619119+0000","last_undegraded":"2024-09-14T04:25:55.619119+0000","last_fullsized":"2024-09-14T04:25:55.619119+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:12.710417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":227,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.576360+0000","last_change":"2024-09-14T04:25:35.560355+0000","last_active":"2024-09-14T04:25:52.576360+0000","last_peered":"2024-09-14T04:25:52.576360+0000","last_clean":"2024-09-14T04:25:52.576360+0000","last_became_active":"2024-09-14T04:25:35.559975+0000","last_became_peered":"2024-09-14T04:25:35.559975+0000","last_unstale":"2024-09-14T04:25:52.576360+0000","last_undegraded":"2024-09-14T04:25:52.576360+0000","last_fullsized":"2024-09-14T04:25:52.576360+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:58.033714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"423'4557","reported_seq":4624,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:57.171577+0000","last_change":"2024-09-14T04:25:07.966414+0000","last_active":"2024-09-14T04:25:57.171577+0000","last_peered":"2024-09-14T04:25:57.171577+0000","last_clean":"2024-09-14T04:25:57.171577+0000","last_became_active":"2024-09-14T04:25:07.966014+0000","last_became_peered":"2024-09-14T04:25:07.966014+0000","last_unstale":"2024-09-14T04:25:57.171577+0000","last_undegraded":"2024-09-14T04:25:57.171577+0000","last_fullsized":"2024-09-14T04:25:57.171577+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:58.062778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298647552,"num_objects":4557,"num_object_clones":0,"num_object_copies":13671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4557,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":910,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624499+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:25:55.624499+0000","last_peered":"2024-09-14T04:25:55.624499+0000","last_clean":"2024-09-14T04:25:55.624499+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:25:55.624499+0000","last_undegraded":"2024-09-14T04:25:55.624499+0000","last_fullsized":"2024-09-14T04:25:55.624499+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":388,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.625251+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:25:55.625251+0000","last_peered":"2024-09-14T04:25:55.625251+0000","last_clean":"2024-09-14T04:25:55.625251+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:25:55.625251+0000","last_undegraded":"2024-09-14T04:25:55.625251+0000","last_fullsized":"2024-09-14T04:25:55.625251+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":694,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620675+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:25:55.620675+0000","last_peered":"2024-09-14T04:25:55.620675+0000","last_clean":"2024-09-14T04:25:55.620675+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:25:55.620675+0000","last_undegraded":"2024-09-14T04:25:55.620675+0000","last_fullsized":"2024-09-14T04:25:55.620675+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":456,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624359+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:25:55.624359+0000","last_peered":"2024-09-14T04:25:55.624359+0000","last_clean":"2024-09-14T04:25:55.624359+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:25:55.624359+0000","last_undegraded":"2024-09-14T04:25:55.624359+0000","last_fullsized":"2024-09-14T04:25:55.624359+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"423'4876","reported_seq":4910,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:01.016501+0000","last_change":"2024-09-14T04:25:01.992854+0000","last_active":"2024-09-14T04:26:01.016501+0000","last_peered":"2024-09-14T04:26:01.016501+0000","last_clean":"2024-09-14T04:26:01.016501+0000","last_became_active":"2024-09-14T04:25:01.992514+0000","last_became_peered":"2024-09-14T04:25:01.992514+0000","last_unstale":"2024-09-14T04:26:01.016501+0000","last_undegraded":"2024-09-14T04:26:01.016501+0000","last_fullsized":"2024-09-14T04:26:01.016501+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4876,"log_dups_size":0,"ondisk_log_size":4876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:59.789126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319553536,"num_objects":4876,"num_object_clones":0,"num_object_copies":14628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4876,"num_write_kb":312064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":662,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621030+0000","last_change":"2024-09-14T04:25:10.213700+0000","last_active":"2024-09-14T04:25:55.621030+0000","last_peered":"2024-09-14T04:25:55.621030+0000","last_clean":"2024-09-14T04:25:55.621030+0000","last_became_active":"2024-09-14T04:25:10.213291+0000","last_became_peered":"2024-09-14T04:25:10.213291+0000","last_unstale":"2024-09-14T04:25:55.621030+0000","last_undegraded":"2024-09-14T04:25:55.621030+0000","last_fullsized":"2024-09-14T04:25:55.621030+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:58.394641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":661,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.619888+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:25:55.619888+0000","last_peered":"2024-09-14T04:25:55.619888+0000","last_clean":"2024-09-14T04:25:55.619888+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:25:55.619888+0000","last_undegraded":"2024-09-14T04:25:55.619888+0000","last_fullsized":"2024-09-14T04:25:55.619888+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":693,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.625854+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:25:55.625854+0000","last_peered":"2024-09-14T04:25:55.625854+0000","last_clean":"2024-09-14T04:25:55.625854+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:25:55.625854+0000","last_undegraded":"2024-09-14T04:25:55.625854+0000","last_fullsized":"2024-09-14T04:25:55.625854+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":161,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.625159+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:25:55.625159+0000","last_peered":"2024-09-14T04:25:55.625159+0000","last_clean":"2024-09-14T04:25:55.625159+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:25:55.625159+0000","last_undegraded":"2024-09-14T04:25:55.625159+0000","last_fullsized":"2024-09-14T04:25:55.625159+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"423'4935","reported_seq":4969,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:00.026844+0000","last_change":"2024-09-14T04:25:02.146113+0000","last_active":"2024-09-14T04:26:00.026844+0000","last_peered":"2024-09-14T04:26:00.026844+0000","last_clean":"2024-09-14T04:26:00.026844+0000","last_became_active":"2024-09-14T04:25:02.145762+0000","last_became_peered":"2024-09-14T04:25:02.145762+0000","last_unstale":"2024-09-14T04:26:00.026844+0000","last_undegraded":"2024-09-14T04:26:00.026844+0000","last_fullsized":"2024-09-14T04:26:00.026844+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:33.000554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4935,"num_write_kb":315840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":650,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569710+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:25:52.569710+0000","last_peered":"2024-09-14T04:25:52.569710+0000","last_clean":"2024-09-14T04:25:52.569710+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:25:52.569710+0000","last_undegraded":"2024-09-14T04:25:52.569710+0000","last_fullsized":"2024-09-14T04:25:52.569710+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":358,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.573847+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:25:52.573847+0000","last_peered":"2024-09-14T04:25:52.573847+0000","last_clean":"2024-09-14T04:25:52.573847+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:25:52.573847+0000","last_undegraded":"2024-09-14T04:25:52.573847+0000","last_fullsized":"2024-09-14T04:25:52.573847+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":441,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575716+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:25:52.575716+0000","last_peered":"2024-09-14T04:25:52.575716+0000","last_clean":"2024-09-14T04:25:52.575716+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:25:52.575716+0000","last_undegraded":"2024-09-14T04:25:52.575716+0000","last_fullsized":"2024-09-14T04:25:52.575716+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":378,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.571331+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:25:52.571331+0000","last_peered":"2024-09-14T04:25:52.571331+0000","last_clean":"2024-09-14T04:25:52.571331+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:25:52.571331+0000","last_undegraded":"2024-09-14T04:25:52.571331+0000","last_fullsized":"2024-09-14T04:25:52.571331+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"423'5010","reported_seq":5044,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:00.985277+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:26:00.985277+0000","last_peered":"2024-09-14T04:26:00.985277+0000","last_clean":"2024-09-14T04:26:00.985277+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:26:00.985277+0000","last_undegraded":"2024-09-14T04:26:00.985277+0000","last_fullsized":"2024-09-14T04:26:00.985277+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5010,"num_write_kb":320640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":682,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615574+0000","last_change":"2024-09-14T04:25:38.633170+0000","last_active":"2024-09-14T04:25:55.615574+0000","last_peered":"2024-09-14T04:25:55.615574+0000","last_clean":"2024-09-14T04:25:55.615574+0000","last_became_active":"2024-09-14T04:25:38.632816+0000","last_became_peered":"2024-09-14T04:25:38.632816+0000","last_unstale":"2024-09-14T04:25:55.615574+0000","last_undegraded":"2024-09-14T04:25:55.615574+0000","last_fullsized":"2024-09-14T04:25:55.615574+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:55:39.218376+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":387,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569746+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:25:52.569746+0000","last_peered":"2024-09-14T04:25:52.569746+0000","last_clean":"2024-09-14T04:25:52.569746+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:25:52.569746+0000","last_undegraded":"2024-09-14T04:25:52.569746+0000","last_fullsized":"2024-09-14T04:25:52.569746+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":750,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623590+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:25:55.623590+0000","last_peered":"2024-09-14T04:25:55.623590+0000","last_clean":"2024-09-14T04:25:55.623590+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:25:55.623590+0000","last_undegraded":"2024-09-14T04:25:55.623590+0000","last_fullsized":"2024-09-14T04:25:55.623590+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":176,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623812+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:25:55.623812+0000","last_peered":"2024-09-14T04:25:55.623812+0000","last_clean":"2024-09-14T04:25:55.623812+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:25:55.623812+0000","last_undegraded":"2024-09-14T04:25:55.623812+0000","last_fullsized":"2024-09-14T04:25:55.623812+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"423'4572","reported_seq":4633,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:57.174442+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:25:57.174442+0000","last_peered":"2024-09-14T04:25:57.174442+0000","last_clean":"2024-09-14T04:25:57.174442+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:25:57.174442+0000","last_undegraded":"2024-09-14T04:25:57.174442+0000","last_fullsized":"2024-09-14T04:25:57.174442+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4572,"log_dups_size":0,"ondisk_log_size":4572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:13.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299630592,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4572,"num_write_kb":292608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":699,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623802+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:25:55.623802+0000","last_peered":"2024-09-14T04:25:55.623802+0000","last_clean":"2024-09-14T04:25:55.623802+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:25:55.623802+0000","last_undegraded":"2024-09-14T04:25:55.623802+0000","last_fullsized":"2024-09-14T04:25:55.623802+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":696,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623488+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:25:55.623488+0000","last_peered":"2024-09-14T04:25:55.623488+0000","last_clean":"2024-09-14T04:25:55.623488+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:25:55.623488+0000","last_undegraded":"2024-09-14T04:25:55.623488+0000","last_fullsized":"2024-09-14T04:25:55.623488+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":427,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.622045+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:25:55.622045+0000","last_peered":"2024-09-14T04:25:55.622045+0000","last_clean":"2024-09-14T04:25:55.622045+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:25:55.622045+0000","last_undegraded":"2024-09-14T04:25:55.622045+0000","last_fullsized":"2024-09-14T04:25:55.622045+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":461,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621771+0000","last_change":"2024-09-14T04:24:08.989110+0000","last_active":"2024-09-14T04:25:55.621771+0000","last_peered":"2024-09-14T04:25:55.621771+0000","last_clean":"2024-09-14T04:25:55.621771+0000","last_became_active":"2024-09-14T04:24:08.987440+0000","last_became_peered":"2024-09-14T04:24:08.987440+0000","last_unstale":"2024-09-14T04:25:55.621771+0000","last_undegraded":"2024-09-14T04:25:55.621771+0000","last_fullsized":"2024-09-14T04:25:55.621771+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:17:32.812292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"423'4823","reported_seq":4857,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:01.021137+0000","last_change":"2024-09-14T04:25:01.993057+0000","last_active":"2024-09-14T04:26:01.021137+0000","last_peered":"2024-09-14T04:26:01.021137+0000","last_clean":"2024-09-14T04:26:01.021137+0000","last_became_active":"2024-09-14T04:25:01.992668+0000","last_became_peered":"2024-09-14T04:25:01.992668+0000","last_unstale":"2024-09-14T04:26:01.021137+0000","last_undegraded":"2024-09-14T04:26:01.021137+0000","last_fullsized":"2024-09-14T04:26:01.021137+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4823,"log_dups_size":0,"ondisk_log_size":4823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:38.544577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4823,"num_write_kb":308672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":728,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575527+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:25:52.575527+0000","last_peered":"2024-09-14T04:25:52.575527+0000","last_clean":"2024-09-14T04:25:52.575527+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:25:52.575527+0000","last_undegraded":"2024-09-14T04:25:52.575527+0000","last_fullsized":"2024-09-14T04:25:52.575527+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":350,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621128+0000","last_change":"2024-09-14T04:24:08.999455+0000","last_active":"2024-09-14T04:25:55.621128+0000","last_peered":"2024-09-14T04:25:55.621128+0000","last_clean":"2024-09-14T04:25:55.621128+0000","last_became_active":"2024-09-14T04:24:08.994093+0000","last_became_peered":"2024-09-14T04:24:08.994093+0000","last_unstale":"2024-09-14T04:25:55.621128+0000","last_undegraded":"2024-09-14T04:25:55.621128+0000","last_fullsized":"2024-09-14T04:25:55.621128+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:29:31.948482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":681,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.623272+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:25:55.623272+0000","last_peered":"2024-09-14T04:25:55.623272+0000","last_clean":"2024-09-14T04:25:55.623272+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:25:55.623272+0000","last_undegraded":"2024-09-14T04:25:55.623272+0000","last_fullsized":"2024-09-14T04:25:55.623272+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":380,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621528+0000","last_change":"2024-09-14T04:25:35.562908+0000","last_active":"2024-09-14T04:25:55.621528+0000","last_peered":"2024-09-14T04:25:55.621528+0000","last_clean":"2024-09-14T04:25:55.621528+0000","last_became_active":"2024-09-14T04:25:35.562023+0000","last_became_peered":"2024-09-14T04:25:35.562023+0000","last_unstale":"2024-09-14T04:25:55.621528+0000","last_undegraded":"2024-09-14T04:25:55.621528+0000","last_fullsized":"2024-09-14T04:25:55.621528+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:55.656734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.8","version":"423'4619","reported_seq":4680,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:57.185112+0000","last_change":"2024-09-14T04:25:01.996105+0000","last_active":"2024-09-14T04:25:57.185112+0000","last_peered":"2024-09-14T04:25:57.185112+0000","last_clean":"2024-09-14T04:25:57.185112+0000","last_became_active":"2024-09-14T04:25:01.995811+0000","last_became_peered":"2024-09-14T04:25:01.995811+0000","last_unstale":"2024-09-14T04:25:57.185112+0000","last_undegraded":"2024-09-14T04:25:57.185112+0000","last_fullsized":"2024-09-14T04:25:57.185112+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:04.759721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4619,"num_write_kb":295616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":971,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615660+0000","last_change":"2024-09-14T04:25:28.605221+0000","last_active":"2024-09-14T04:25:55.615660+0000","last_peered":"2024-09-14T04:25:55.615660+0000","last_clean":"2024-09-14T04:25:55.615660+0000","last_became_active":"2024-09-14T04:25:28.604819+0000","last_became_peered":"2024-09-14T04:25:28.604819+0000","last_unstale":"2024-09-14T04:25:55.615660+0000","last_undegraded":"2024-09-14T04:25:55.615660+0000","last_fullsized":"2024-09-14T04:25:55.615660+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:21.033138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":460,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624193+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:25:55.624193+0000","last_peered":"2024-09-14T04:25:55.624193+0000","last_clean":"2024-09-14T04:25:55.624193+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:25:55.624193+0000","last_undegraded":"2024-09-14T04:25:55.624193+0000","last_fullsized":"2024-09-14T04:25:55.624193+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":526,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620449+0000","last_change":"2024-09-14T04:23:27.596032+0000","last_active":"2024-09-14T04:25:55.620449+0000","last_peered":"2024-09-14T04:25:55.620449+0000","last_clean":"2024-09-14T04:25:55.620449+0000","last_became_active":"2024-09-14T04:23:27.595521+0000","last_became_peered":"2024-09-14T04:23:27.595521+0000","last_unstale":"2024-09-14T04:25:55.620449+0000","last_undegraded":"2024-09-14T04:25:55.620449+0000","last_fullsized":"2024-09-14T04:25:55.620449+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:14.145527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":407,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575182+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:25:52.575182+0000","last_peered":"2024-09-14T04:25:52.575182+0000","last_clean":"2024-09-14T04:25:52.575182+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:25:52.575182+0000","last_undegraded":"2024-09-14T04:25:52.575182+0000","last_fullsized":"2024-09-14T04:25:52.575182+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":444,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575607+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:25:52.575607+0000","last_peered":"2024-09-14T04:25:52.575607+0000","last_clean":"2024-09-14T04:25:52.575607+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:25:52.575607+0000","last_undegraded":"2024-09-14T04:25:52.575607+0000","last_fullsized":"2024-09-14T04:25:52.575607+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":443,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575381+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:25:52.575381+0000","last_peered":"2024-09-14T04:25:52.575381+0000","last_clean":"2024-09-14T04:25:52.575381+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:25:52.575381+0000","last_undegraded":"2024-09-14T04:25:52.575381+0000","last_fullsized":"2024-09-14T04:25:52.575381+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":147,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624160+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:25:55.624160+0000","last_peered":"2024-09-14T04:25:55.624160+0000","last_clean":"2024-09-14T04:25:55.624160+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:25:55.624160+0000","last_undegraded":"2024-09-14T04:25:55.624160+0000","last_fullsized":"2024-09-14T04:25:55.624160+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":365,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616412+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:25:55.616412+0000","last_peered":"2024-09-14T04:25:55.616412+0000","last_clean":"2024-09-14T04:25:55.616412+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:25:55.616412+0000","last_undegraded":"2024-09-14T04:25:55.616412+0000","last_fullsized":"2024-09-14T04:25:55.616412+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":906,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620082+0000","last_change":"2024-09-14T04:25:10.214026+0000","last_active":"2024-09-14T04:25:55.620082+0000","last_peered":"2024-09-14T04:25:55.620082+0000","last_clean":"2024-09-14T04:25:55.620082+0000","last_became_active":"2024-09-14T04:25:10.212511+0000","last_became_peered":"2024-09-14T04:25:10.212511+0000","last_unstale":"2024-09-14T04:25:55.620082+0000","last_undegraded":"2024-09-14T04:25:55.620082+0000","last_fullsized":"2024-09-14T04:25:55.620082+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:20.633170+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":314,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570175+0000","last_change":"2024-09-14T04:25:35.559702+0000","last_active":"2024-09-14T04:25:52.570175+0000","last_peered":"2024-09-14T04:25:52.570175+0000","last_clean":"2024-09-14T04:25:52.570175+0000","last_became_active":"2024-09-14T04:25:35.559395+0000","last_became_peered":"2024-09-14T04:25:35.559395+0000","last_unstale":"2024-09-14T04:25:52.570175+0000","last_undegraded":"2024-09-14T04:25:52.570175+0000","last_fullsized":"2024-09-14T04:25:52.570175+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:51.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":304,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.569909+0000","last_change":"2024-09-14T04:24:09.122059+0000","last_active":"2024-09-14T04:25:52.569909+0000","last_peered":"2024-09-14T04:25:52.569909+0000","last_clean":"2024-09-14T04:25:52.569909+0000","last_became_active":"2024-09-14T04:24:09.121824+0000","last_became_peered":"2024-09-14T04:24:09.121824+0000","last_unstale":"2024-09-14T04:25:52.569909+0000","last_undegraded":"2024-09-14T04:25:52.569909+0000","last_fullsized":"2024-09-14T04:25:52.569909+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:36.179815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":455,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621256+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:25:55.621256+0000","last_peered":"2024-09-14T04:25:55.621256+0000","last_clean":"2024-09-14T04:25:55.621256+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:25:55.621256+0000","last_undegraded":"2024-09-14T04:25:55.621256+0000","last_fullsized":"2024-09-14T04:25:55.621256+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":559,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615405+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:25:55.615405+0000","last_peered":"2024-09-14T04:25:55.615405+0000","last_clean":"2024-09-14T04:25:55.615405+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:25:55.615405+0000","last_undegraded":"2024-09-14T04:25:55.615405+0000","last_fullsized":"2024-09-14T04:25:55.615405+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":136,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570271+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:25:52.570271+0000","last_peered":"2024-09-14T04:25:52.570271+0000","last_clean":"2024-09-14T04:25:52.570271+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:25:52.570271+0000","last_undegraded":"2024-09-14T04:25:52.570271+0000","last_fullsized":"2024-09-14T04:25:52.570271+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.612161+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:25:55.612161+0000","last_peered":"2024-09-14T04:25:55.612161+0000","last_clean":"2024-09-14T04:25:55.612161+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:25:55.612161+0000","last_undegraded":"2024-09-14T04:25:55.612161+0000","last_fullsized":"2024-09-14T04:25:55.612161+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":471,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.624085+0000","last_change":"2024-09-14T04:25:35.564324+0000","last_active":"2024-09-14T04:25:55.624085+0000","last_peered":"2024-09-14T04:25:55.624085+0000","last_clean":"2024-09-14T04:25:55.624085+0000","last_became_active":"2024-09-14T04:25:35.562918+0000","last_became_peered":"2024-09-14T04:25:35.562918+0000","last_unstale":"2024-09-14T04:25:55.624085+0000","last_undegraded":"2024-09-14T04:25:55.624085+0000","last_fullsized":"2024-09-14T04:25:55.624085+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:16.172185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":719,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.622911+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:25:55.622911+0000","last_peered":"2024-09-14T04:25:55.622911+0000","last_clean":"2024-09-14T04:25:55.622911+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:25:55.622911+0000","last_undegraded":"2024-09-14T04:25:55.622911+0000","last_fullsized":"2024-09-14T04:25:55.622911+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":272,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.570341+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:25:52.570341+0000","last_peered":"2024-09-14T04:25:52.570341+0000","last_clean":"2024-09-14T04:25:52.570341+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:25:52.570341+0000","last_undegraded":"2024-09-14T04:25:52.570341+0000","last_fullsized":"2024-09-14T04:25:52.570341+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":12,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.071461+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:25:55.071461+0000","last_peered":"2024-09-14T04:25:55.071461+0000","last_clean":"2024-09-14T04:25:55.071461+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:25:55.071461+0000","last_undegraded":"2024-09-14T04:25:55.071461+0000","last_fullsized":"2024-09-14T04:25:55.071461+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":130,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.620186+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:25:55.620186+0000","last_peered":"2024-09-14T04:25:55.620186+0000","last_clean":"2024-09-14T04:25:55.620186+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:25:55.620186+0000","last_undegraded":"2024-09-14T04:25:55.620186+0000","last_fullsized":"2024-09-14T04:25:55.620186+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":469,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621672+0000","last_change":"2024-09-14T04:23:27.573928+0000","last_active":"2024-09-14T04:25:55.621672+0000","last_peered":"2024-09-14T04:25:55.621672+0000","last_clean":"2024-09-14T04:25:55.621672+0000","last_became_active":"2024-09-14T04:23:27.573557+0000","last_became_peered":"2024-09-14T04:23:27.573557+0000","last_unstale":"2024-09-14T04:25:55.621672+0000","last_undegraded":"2024-09-14T04:25:55.621672+0000","last_fullsized":"2024-09-14T04:25:55.621672+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:07.509590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":32,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575178+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:25:52.575178+0000","last_peered":"2024-09-14T04:25:52.575178+0000","last_clean":"2024-09-14T04:25:52.575178+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:25:52.575178+0000","last_undegraded":"2024-09-14T04:25:52.575178+0000","last_fullsized":"2024-09-14T04:25:52.575178+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":370,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.572430+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:25:52.572430+0000","last_peered":"2024-09-14T04:25:52.572430+0000","last_clean":"2024-09-14T04:25:52.572430+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:25:52.572430+0000","last_undegraded":"2024-09-14T04:25:52.572430+0000","last_fullsized":"2024-09-14T04:25:52.572430+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.575180+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:25:52.575180+0000","last_peered":"2024-09-14T04:25:52.575180+0000","last_clean":"2024-09-14T04:25:52.575180+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:25:52.575180+0000","last_undegraded":"2024-09-14T04:25:52.575180+0000","last_fullsized":"2024-09-14T04:25:52.575180+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":379,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.606313+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:25:52.606313+0000","last_peered":"2024-09-14T04:25:52.606313+0000","last_clean":"2024-09-14T04:25:52.606313+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:25:52.606313+0000","last_undegraded":"2024-09-14T04:25:52.606313+0000","last_fullsized":"2024-09-14T04:25:52.606313+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":608,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.619728+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:25:55.619728+0000","last_peered":"2024-09-14T04:25:55.619728+0000","last_clean":"2024-09-14T04:25:55.619728+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:25:55.619728+0000","last_undegraded":"2024-09-14T04:25:55.619728+0000","last_fullsized":"2024-09-14T04:25:55.619728+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":279,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.619653+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:25:55.619653+0000","last_peered":"2024-09-14T04:25:55.619653+0000","last_clean":"2024-09-14T04:25:55.619653+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:25:55.619653+0000","last_undegraded":"2024-09-14T04:25:55.619653+0000","last_fullsized":"2024-09-14T04:25:55.619653+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":404,"reported_epoch":422,"state":"active+clean","last_fresh":"2024-09-14T04:25:52.574913+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:25:52.574913+0000","last_peered":"2024-09-14T04:25:52.574913+0000","last_clean":"2024-09-14T04:25:52.574913+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:25:52.574913+0000","last_undegraded":"2024-09-14T04:25:52.574913+0000","last_fullsized":"2024-09-14T04:25:52.574913+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":503,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.621262+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:25:55.621262+0000","last_peered":"2024-09-14T04:25:55.621262+0000","last_clean":"2024-09-14T04:25:55.621262+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:25:55.621262+0000","last_undegraded":"2024-09-14T04:25:55.621262+0000","last_fullsized":"2024-09-14T04:25:55.621262+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5063966720,"num_objects":77270,"num_object_clones":0,"num_object_copies":231810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1840,"num_objects_unfound":0,"num_objects_dirty":77270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":77270,"num_write_kb":4945280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3534,"num_bytes_recovered":231604224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15359082496,"data_stored":15359082496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77270,"ondisk_log_size":77270,"up":48,"acting":47,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":12322,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2506969,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201825,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463463,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834136,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3629764,"kb_used_data":3453884,"kb_used_omap":5,"kb_used_meta":175866,"kb_avail":90115388,"statfs":{"total":95995035648,"available":92278157312,"internally_reserved":0,"allocated":3536777216,"data_stored":3534035282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":180086845},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986735,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4435408,"kb_used_data":3113160,"kb_used_omap":1,"kb_used_meta":1322238,"kb_avail":89309744,"statfs":{"total":95995035648,"available":91453177856,"internally_reserved":0,"allocated":3187875840,"data_stored":3185125383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1353972166},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748763,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2720328,"kb_used_data":1770620,"kb_used_omap":4,"kb_used_meta":949691,"kb_avail":91024824,"statfs":{"total":95995035648,"available":93209419776,"internally_reserved":0,"allocated":1813114880,"data_stored":1810372728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":972484140},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286984,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3091792,"kb_used_data":2653064,"kb_used_omap":10,"kb_used_meta":438709,"kb_avail":90653360,"statfs":{"total":95995035648,"available":92829040640,"internally_reserved":0,"allocated":2716737536,"data_stored":2713988308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":449238615},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468307,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2905212,"kb_used_data":1895924,"kb_used_omap":5,"kb_used_meta":1009274,"kb_avail":90839940,"statfs":{"total":95995035648,"available":93020098560,"internally_reserved":0,"allocated":1941426176,"data_stored":1938690133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1033496927},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342422,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678296,"kb_used_data":2043088,"kb_used_omap":10,"kb_used_meta":635189,"kb_avail":91066856,"statfs":{"total":95995035648,"available":93252460544,"internally_reserved":0,"allocated":2092122112,"data_stored":2089364279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":650433746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2098921472,"data_stored":2098921472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1950154752,"data_stored":1950154752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2741501952,"data_stored":2741501952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1817313280,"data_stored":1817313280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3182690304,"data_stored":3182690304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3568500736,"data_stored":3568500736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:26:04.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:26:04.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:04.127 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:04.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:04.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:26:04.229 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:04.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:04.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:04.472 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:04.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:04.613 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:04.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:04.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:04.751 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:04.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 83161 83145 94.457 67.5 0.0179305 0.0105818 2024-09-14T04:26:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:04 smithi012 ceph-mon[101180]: from='client.46136 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:04 smithi012 ceph-mon[101180]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1840/237156 objects misplaced (0.776%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:04.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:04.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:04.906 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:04.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:04.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[107399]: from='client.46136 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[107399]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1840/237156 objects misplaced (0.776%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[112165]: from='client.46136 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[112165]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1840/237156 objects misplaced (0.776%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:04.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:05.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:05.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:05.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:05.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:05.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:05.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.285 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:05.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:05.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:05.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:05.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:05.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:05.649 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:05.694 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:05.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:05.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 84010 83994 93.7179 53.0625 0.00956122 0.010665 2024-09-14T04:26:05.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:05.835 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:05.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:05.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:05.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:05.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:06.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:06.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:06.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:06.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:06.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:06.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[112165]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s; 1840/242520 objects misplaced (0.759%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:06.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:05 smithi001 ceph-mon[107399]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s; 1840/242520 objects misplaced (0.759%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:06.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:06.342 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:05 smithi012 ceph-mon[101180]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s; 1840/242520 objects misplaced (0.759%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:26:06.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:06.487 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:06.534 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:06.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:06.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:06.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:06.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:06.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:06.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 85160 85144 93.3346 71.875 0.00778919 0.0107092 2024-09-14T04:26:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:06.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:06.896 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:06] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:26:06.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:06.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:06.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:07.042 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:07.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:07.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:07.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:07.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.347 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:07.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:07.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:07.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:07.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:07.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:07.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:07.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:07.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:07.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 86271 86255 92.9226 69.4375 0.00885922 0.0107569 2024-09-14T04:26:07.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:07.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:07.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:08.018 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:08.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:08.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:08.199 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:08.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:08.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:08.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:08.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:08.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:08.601 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:08.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:08.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:08 smithi001 ceph-mon[107399]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 1589/253296 objects misplaced (0.627%); 4.2 MiB/s, 66 objects/s recovering 2024-09-14T04:26:08.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:08 smithi001 ceph-mon[112165]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 1589/253296 objects misplaced (0.627%); 4.2 MiB/s, 66 objects/s recovering 2024-09-14T04:26:08.760 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:08.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 87393 87377 92.5361 70.125 0.0117286 0.0108019 2024-09-14T04:26:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:08 smithi012 ceph-mon[101180]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 1589/253296 objects misplaced (0.627%); 4.2 MiB/s, 66 objects/s recovering 2024-09-14T04:26:08.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:08.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:08.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:09.001 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:09.001 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:09.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:09.078 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:09.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:09.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:09.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.361 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145641 12.9 12:9c4f5bd4:::benchmark_data_smithi001_255951_object145640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 381.91416289300003, 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4931161319999999, 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145641 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.362 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441856+0000", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441857+0000", 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.615e-06 2024-09-14T04:26:09.363 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441881+0000", 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4358e-05 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441891+0000", 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1370000000000008e-06 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.364 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441899+0000", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4109999999999996e-06 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.442318+0000", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00041919 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.442327+0000", 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3850000000000002e-06 2024-09-14T04:26:09.365 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.898231+0000", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.455904879 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.900067+0000", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0018354840000000001 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.900169+0000", 2024-09-14T04:26:09.366 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102407 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.902502+0000", 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023330870000000002 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.903274+0000", 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00077130899999999997 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.367 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934788+0000", 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.031514693000000003 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934872+0000", 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3610999999999996e-05 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.934972+0000", 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9566000000000002e-05 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.368 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243106 12.4 12:24c6f3b1:::benchmark_data_smithi001_255951_object97446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 241.78466176699999, 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.79898594000000001, 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243106 2024-09-14T04:26:09.369 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571357+0000", 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.370 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571358+0000", 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4050000000000001e-06 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571364+0000", 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9059999999999999e-06 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571378+0000", 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3919e-05 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.371 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571384+0000", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0950000000000001e-06 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571447+0000", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2761000000000004e-05 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.571454+0000", 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1380000000000001e-06 2024-09-14T04:26:09.372 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.357976+0000", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78652253299999997 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.358107+0000", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013065900000000001 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.358238+0000", 2024-09-14T04:26:09.373 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013079300000000001 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.362227+0000", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0039890630000000002 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370247+0000", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080196199999999999 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370256+0000", 2024-09-14T04:26:09.374 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8689999999999999e-06 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370304+0000", 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7994000000000002e-05 2024-09-14T04:26:09.375 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:08.370343+0000", 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8185000000000001e-05 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243551 12.4 12:26642039:::benchmark_data_smithi001_255951_object97891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 238.81320748300001, 2024-09-14T04:26:09.376 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4444095940000001, 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243551 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.377 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542811+0000", 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542813+0000", 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.809e-06 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.378 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542820+0000", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6129999999999998e-06 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542845+0000", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4627999999999999e-05 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542855+0000", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.041e-05 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.379 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542933+0000", 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8009000000000005e-05 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.542946+0000", 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2379e-05 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317323+0000", 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.77437684399999995 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.380 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317337+0000", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4119e-05 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932714+0000", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6153777499999999 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.932847+0000", 2024-09-14T04:26:09.381 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013261300000000001 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.933426+0000", 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00057883999999999995 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952181+0000", 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018755323000000001 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.382 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952240+0000", 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8390999999999997e-05 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952638+0000", 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00039805999999999999 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.986992+0000", 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.034354518000000001 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.383 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987114+0000", 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000121679 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987221+0000", 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000106609 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243553 12.4 12:2f2271ff:::benchmark_data_smithi001_255951_object97893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:26:09.384 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 238.69645938100001, 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.338288382, 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243553 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.385 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659559+0000", 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659562+0000", 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.492e-06 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.386 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659576+0000", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4008999999999999e-05 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659597+0000", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1169e-05 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659607+0000", 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0511e-05 2024-09-14T04:26:09.387 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659683+0000", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5479e-05 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.659695+0000", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2529e-05 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317362+0000", 2024-09-14T04:26:09.388 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.65766659800000005 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317369+0000", 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3599999999999998e-06 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.952702+0000", 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.635333283 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.389 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.970997+0000", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018294331 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.971219+0000", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022208699999999999 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987267+0000", 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.016048271999999999 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.390 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987279+0000", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2177e-05 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987311+0000", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1829000000000003e-05 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997605+0000", 2024-09-14T04:26:09.391 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010293771 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997713+0000", 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108224 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997847+0000", 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013426099999999999 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.392 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243556 12.4 12:21983187:::benchmark_data_smithi001_255951_object97896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 238.45180749400001, 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0939079889999999, 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243556 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.393 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904211+0000", 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.394 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904213+0000", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3609999999999999e-06 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904221+0000", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9589999999999995e-06 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904242+0000", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0871000000000001e-05 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.395 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904252+0000", 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6479999999999998e-06 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904325+0000", 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3492999999999996e-05 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.904338+0000", 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2510000000000001e-05 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.396 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317390+0000", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41305234899999999 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317402+0000", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1776e-05 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.980836+0000", 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6634340249999999 2024-09-14T04:26:09.397 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.980939+0000", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103546 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.981063+0000", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012321599999999999 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987347+0000", 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062843559999999996 2024-09-14T04:26:09.398 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987385+0000", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8389999999999997e-05 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.995080+0000", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0076942310000000002 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997925+0000", 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028455440000000002 2024-09-14T04:26:09.399 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998016+0000", 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0470000000000006e-05 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998119+0000", 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103244 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.400 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243559 12.4 12:2d42a702:::benchmark_data_smithi001_255951_object97899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 238.30733015300001, 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.950219755, 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243559 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.401 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048688+0000", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048690+0000", 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9259999999999999e-06 2024-09-14T04:26:09.402 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048697+0000", 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2930000000000003e-06 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.403 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048716+0000", 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8921999999999999e-05 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048724+0000", 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4499999999999998e-06 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048781+0000", 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6925999999999997e-05 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.404 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.048792+0000", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1717e-05 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317428+0000", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.26863533699999997 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.317446+0000", 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7995e-05 2024-09-14T04:26:09.405 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987414+0000", 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.669967846 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987529+0000", 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115388 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:26:09.406 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.987651+0000", 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000122014 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.995878+0000", 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0082268069999999992 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.997223+0000", 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001345197 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.407 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998455+0000", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012323010000000001 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998493+0000", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7530999999999997e-05 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998580+0000", 2024-09-14T04:26:09.408 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7101999999999994e-05 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.998908+0000", 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000328003 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272408 12.4 12:25b358f7:::benchmark_data_smithi001_255951_object126748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.60580079299999, 2024-09-14T04:26:09.409 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99180387999999997, 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272408 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:26:09.410 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750219+0000", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6390000000000001e-06 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750225+0000", 2024-09-14T04:26:09.411 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3650000000000003e-06 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750237+0000", 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2736e-05 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750244+0000", 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.109e-06 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.412 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750439+0000", 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000194601 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750448+0000", 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8799999999999997e-06 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732347+0000", 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98189893699999997 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.413 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732448+0000", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101087 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732831+0000", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038299800000000001 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737544+0000", 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047135329999999998 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.414 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741660+0000", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041152360000000004 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741800+0000", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013995699999999999 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741974+0000", 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000174733 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.415 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742021+0000", 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7068999999999999e-05 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272413 12.4 12:200feb18:::benchmark_data_smithi001_255951_object126753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.60052884000001, 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98673433499999996, 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.416 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272413 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.417 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:26:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:09.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.419 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755491+0000", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6619999999999999e-06 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755496+0000", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5220000000000001e-06 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755511+0000", 2024-09-14T04:26:09.420 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4866e-05 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755518+0000", 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.046e-06 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755570+0000", 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2743e-05 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.421 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755583+0000", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2396e-05 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735225+0000", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97964242499999998 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735341+0000", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116228 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.422 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735690+0000", 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000348871 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741614+0000", 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059240389999999999 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742089+0000", 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00047413899999999999 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.423 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742126+0000", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7982999999999999e-05 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742181+0000", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4051000000000001e-05 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742224+0000", 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3364e-05 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.424 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272414 12.4 12:202c47cc:::benchmark_data_smithi001_255951_object126754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.59867911399999, 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98523925499999998, 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.425 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272414 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.426 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757340+0000", 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.102e-06 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757344+0000", 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8759999999999998e-06 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757354+0000", 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7929999999999992e-06 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.427 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757359+0000", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9350000000000002e-06 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757399+0000", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9684999999999997e-05 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757404+0000", 2024-09-14T04:26:09.428 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1560000000000003e-06 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737575+0000", 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98017141100000005 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737674+0000", 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8740000000000007e-05 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.429 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.738014+0000", 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00033988699999999998 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742033+0000", 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0040193670000000003 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742282+0000", 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024841899999999999 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.430 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742345+0000", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3440000000000002e-05 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742480+0000", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013516099999999999 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742579+0000", 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8282999999999994e-05 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.431 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272475 12.4 12:26c69f4d:::benchmark_data_smithi001_255951_object126815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.45016934399999, 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.84511323999999999, 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272475 2024-09-14T04:26:09.432 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.433 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905850+0000", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6300000000000004e-07 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905854+0000", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1999999999999996e-06 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905864+0000", 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3980000000000005e-06 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.434 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905870+0000", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5899999999999996e-06 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.906017+0000", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000146754 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.906027+0000", 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0481e-05 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.739685+0000", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.83365730299999996 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.739814+0000", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012941100000000001 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.740118+0000", 2024-09-14T04:26:09.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000304157 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742297+0000", 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021787550000000001 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742652+0000", 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035510799999999998 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750860+0000", 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0082081570000000006 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750917+0000", 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7090999999999999e-05 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750962+0000", 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4972000000000002e-05 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.438 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272526 12.4 12:235239d5:::benchmark_data_smithi001_255951_object126866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.35183596799999, 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.74694861300000004, 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.439 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272526 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004184+0000", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9180000000000001e-06 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004191+0000", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0040000000000002e-06 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004205+0000", 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.34e-05 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004213+0000", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1359999999999997e-06 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004884+0000", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00067096400000000002 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004894+0000", 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8309999999999996e-06 2024-09-14T04:26:09.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742737+0000", 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.73784364099999999 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742850+0000", 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113114 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.743167+0000", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00031664200000000001 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750977+0000", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0078100720000000004 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751016+0000", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9014000000000003e-05 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751057+0000", 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1177999999999998e-05 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751098+0000", 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1044999999999998e-05 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751131+0000", 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2654000000000003e-05 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272546 12.4 12:228731de:::benchmark_data_smithi001_255951_object126886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 150.32030777099999, 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.723662044, 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.446 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272546 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035713+0000", 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0059999999999999e-06 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035718+0000", 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0839999999999999e-06 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.448 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035732+0000", 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4499e-05 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035739+0000", 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9809999999999997e-06 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035786+0000", 2024-09-14T04:26:09.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6733000000000001e-05 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035794+0000", 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7819999999999994e-06 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.746984+0000", 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.71119057500000005 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.747103+0000", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119184 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.747474+0000", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000370386 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.755439+0000", 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007964647 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759237+0000", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037988399999999999 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759284+0000", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6878000000000001e-05 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759331+0000", 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7261e-05 2024-09-14T04:26:09.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759373+0000", 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1187999999999999e-05 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273651 12.14 12:2adccd9f:::benchmark_data_smithi001_255951_object127991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 148.554242703, 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95309075099999996, 2024-09-14T04:26:09.453 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273651 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801777+0000", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7910000000000001e-06 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801783+0000", 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1080000000000001e-06 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801800+0000", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7554000000000002e-05 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801808+0000", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4359999999999996e-06 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801880+0000", 2024-09-14T04:26:09.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2213999999999997e-05 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801889+0000", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3999999999999998e-06 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.748500+0000", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94661125000000002 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.748641+0000", 2024-09-14T04:26:09.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014093800000000001 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.749078+0000", 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043684399999999998 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754302+0000", 2024-09-14T04:26:09.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052233640000000003 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754347+0000", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5853999999999998e-05 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754530+0000", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018233400000000001 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754811+0000", 2024-09-14T04:26:09.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00028136900000000001 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754866+0000", 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5294999999999998e-05 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273762 12.14 12:2898c893:::benchmark_data_smithi001_255951_object128102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:26:09.460 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 148.38580831900001, 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78739233399999997, 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273762 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970212+0000", 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9120000000000001e-06 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970218+0000", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7860000000000002e-06 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970234+0000", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6537000000000001e-05 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970243+0000", 2024-09-14T04:26:09.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0599999999999997e-06 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970364+0000", 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120578 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970373+0000", 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7129999999999994e-06 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.751673+0000", 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78130018899999998 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.751788+0000", 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115554 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.752301+0000", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00051248099999999996 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757408+0000", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051068470000000003 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757453+0000", 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4897000000000001e-05 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757496+0000", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3368000000000002e-05 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757553+0000", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7448999999999998e-05 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757602+0000", 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8962999999999999e-05 2024-09-14T04:26:09.471 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273780 12.14 12:2f2a76c0:::benchmark_data_smithi001_255951_object128120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 148.353206247, 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.76112474900000004, 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:09.472 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273780 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002814+0000", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7260000000000001e-06 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002819+0000", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.203e-06 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002844+0000", 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5063e-05 2024-09-14T04:26:09.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002850+0000", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.857e-06 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002915+0000", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5147000000000005e-05 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002925+0000", 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0274e-05 2024-09-14T04:26:09.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754907+0000", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.75198182400000002 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.755018+0000", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011041599999999999 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.755399+0000", 2024-09-14T04:26:09.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038100900000000002 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763712+0000", 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0083134530000000002 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763760+0000", 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8164000000000002e-05 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763808+0000", 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.799e-05 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763888+0000", 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9270000000000005e-05 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763937+0000", 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9353e-05 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.478 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:76466 13.c 13:3a84d052:::benchmark_data_smithi001_306460_object76465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:25:58.699559+0000", 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 10.656459247000001, 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.079332589999999995, 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 76466 2024-09-14T04:26:09.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699559+0000", 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699559+0000", 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699560+0000", 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.325e-06 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699585+0000", 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4049999999999998e-05 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699594+0000", 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6479999999999998e-06 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.699601+0000", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3170000000000004e-06 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.702125+0000", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002523183 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.702248+0000", 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123041 2024-09-14T04:26:09.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.702332+0000", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4495999999999999e-05 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.704227+0000", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001894983 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.704292+0000", 2024-09-14T04:26:09.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4930000000000003e-05 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.778593+0000", 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.074300864999999994 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.778839+0000", 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024632899999999999 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:25:58.778892+0000", 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2423000000000003e-05 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83507 13.c 13:334b35ae:::benchmark_data_smithi001_306460_object83506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.2569371470000004, 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23991748900000001, 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.485 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83507 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.486 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099082+0000", 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9500000000000003e-07 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099130+0000", 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8446000000000003e-05 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099141+0000", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1229e-05 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099147+0000", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8119999999999996e-06 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.103543+0000", 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043959130000000004 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.103756+0000", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021284400000000001 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.104153+0000", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000397289 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107757+0000", 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036035279999999999 2024-09-14T04:26:09.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.111968+0000", 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042115579999999998 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338759+0000", 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22679080200000001 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338938+0000", 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000179173 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338999+0000", 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0399999999999998e-05 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83508 13.c 13:3facc2f7:::benchmark_data_smithi001_306460_object83507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:26:09.491 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.2568628679999998, 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.240055885, 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83508 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:26:09.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1399999999999997e-07 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099199+0000", 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2914000000000003e-05 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099209+0000", 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0232e-05 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099221+0000", 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2272999999999999e-05 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107806+0000", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0085844770000000001 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107956+0000", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015019999999999999 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.108108+0000", 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015174700000000001 2024-09-14T04:26:09.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.114878+0000", 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0067705279999999996 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.115865+0000", 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00098683100000000008 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339066+0000", 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.223201183 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339167+0000", 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101035 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339211+0000", 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4051000000000002e-05 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.497 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83512 13.c 13:3f26c0d6:::benchmark_data_smithi001_306460_object83511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.099895+0000", 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.256123552, 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23954101, 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83512 2024-09-14T04:26:09.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099895+0000", 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099895+0000", 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099896+0000", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.95e-07 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099945+0000", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9509999999999999e-05 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099960+0000", 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4562000000000001e-05 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099967+0000", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9009999999999997e-06 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.111999+0000", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012032127 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.112132+0000", 2024-09-14T04:26:09.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000133013 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.112285+0000", 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015337 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.114972+0000", 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026866680000000001 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.116221+0000", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012486369999999999 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339269+0000", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.223048313 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339386+0000", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117315 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339436+0000", 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9598999999999997e-05 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83526 13.c 13:3c0ae88a:::benchmark_data_smithi001_306460_object83525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.119098+0000", 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 4.2369201810000003, 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22052298200000001, 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:09.504 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83526 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119098+0000", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119098+0000", 2024-09-14T04:26:09.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119099+0000", 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.274e-06 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119120+0000", 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0041e-05 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:09.506 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119130+0000", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.063e-05 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119135+0000", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0470000000000001e-06 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119192+0000", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7272999999999999e-05 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:09.507 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119359+0000", 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016667599999999999 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.119541+0000", 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000181454 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.121831+0000", 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022899309999999998 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.122695+0000", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00086457099999999996 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339482+0000", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21678673700000001 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339573+0000", 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1143999999999993e-05 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:09.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339621+0000", 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8204000000000001e-05 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:09.510 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:26:09.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:09.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:09.571 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:26:09.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:09.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2032417650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.685 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.685 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2032417650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.686 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.692 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:09.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:09.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:26:09.796531+0000 min lat: 0.00114104 max lat: 2.49143 avg lat: 0.0108688 2024-09-14T04:26:09.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:26:09.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 16 88322 88306 91.9616 58.0625 0.0139169 0.0108688 2024-09-14T04:26:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:09.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2032417650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:26:09.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:09.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:09.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:10.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:10.115 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:10.217 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:10.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.261 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:10.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:10.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:10.402 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:10.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:10.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:10.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.678 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:10.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:10.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:10.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 89446 89430 91.6056 70.25 0.0192035 0.0109116 2024-09-14T04:26:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.824 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:10.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:10 smithi012 ceph-mon[101180]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 1589/253296 objects misplaced (0.627%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:26:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:10 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:10 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:26:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:10 smithi012 ceph-mon[101180]: osdmap e424: 8 total, 8 up, 6 in 2024-09-14T04:26:10.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:10.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:10.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:10.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:10.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[107399]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 1589/253296 objects misplaced (0.627%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[107399]: osdmap e424: 8 total, 8 up, 6 in 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[112165]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 1589/253296 objects misplaced (0.627%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:26:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:10 smithi001 ceph-mon[112165]: osdmap e424: 8 total, 8 up, 6 in 2024-09-14T04:26:11.069 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:11.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:11.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:11.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:11.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:11.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:11.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:11.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:11.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:11.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:11.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:11.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:11.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:11.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:11.796 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 90403 90387 91.0928 59.8125 0.00504955 0.0109614 2024-09-14T04:26:11.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:11.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:11.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:12.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:12.059 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:12.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:12.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:12.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:12.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:12.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:12.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:12.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:12.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:12.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:12.571 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.619 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:12.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:12.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:12.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:12 smithi001 ceph-mon[112165]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 1308/264417 objects misplaced (0.495%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:12.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:12 smithi001 ceph-mon[107399]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 1308/264417 objects misplaced (0.495%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.770 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:12.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:12.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 91362 91346 90.5982 59.9375 0.0265025 0.0110321 2024-09-14T04:26:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:12 smithi012 ceph-mon[101180]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 1308/264417 objects misplaced (0.495%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:12.872 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:12.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:12.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:12.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:12.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:26:13.015 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:13.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:26:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.088 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:13.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:13.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:13.215 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:13.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.353 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:13.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:13.454 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:13.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:13.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:13.641 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:13.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:13.742 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:13.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:13.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 92351 92335 90.1484 61.8125 0.0127353 0.0110873 2024-09-14T04:26:13.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:13.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:13.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:14.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:14.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:14.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:14.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:14.189 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:14.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:14.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.328 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:14.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:14.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:14.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:14.559 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:14.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:14.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:14 smithi001 ceph-mon[107399]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 1308/269106 objects misplaced (0.486%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:14.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:14 smithi001 ceph-mon[112165]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 1308/269106 objects misplaced (0.486%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:14.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:14.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:14.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 93472 93456 89.8393 70.0625 0.0156899 0.0111265 2024-09-14T04:26:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:14 smithi012 ceph-mon[101180]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 1308/269106 objects misplaced (0.486%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:14.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:14.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:14.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:14.994 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:15.020 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:15.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:15.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.165 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:15.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:15.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:15.301 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:15.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:15.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:15.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:15.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:15.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:15.665 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.714 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:15.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:15.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 94507 94491 89.4582 64.6875 0.024203 0.0111725 2024-09-14T04:26:15.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:15.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:15.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:15.958 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:15.999 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:16.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:16.122 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:16.224 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:16.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:16.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:16.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:16.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:16.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:16.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.697 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:16.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:16.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 95523 95507 89.0707 63.5 0.00301585 0.0112161 2024-09-14T04:26:16.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:16.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:16.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:16 smithi012 ceph-mon[101180]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 1308/274008 objects misplaced (0.477%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:16.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:16.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:16.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:16.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:16 smithi001 ceph-mon[112165]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 1308/274008 objects misplaced (0.477%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:16 smithi001 ceph-mon[107399]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 1308/274008 objects misplaced (0.477%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:26:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:16] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:26:17.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:17.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:17.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:17.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:17.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:17.371 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:17.473 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:17.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:17.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:17.642 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:17.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:17.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:17.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:17.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 96531 96515 88.6873 63 0.0207166 0.0112702 2024-09-14T04:26:17.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:17.920 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:17.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:18.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:18.026 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:18.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:18.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:18.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:18.291 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:18.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:18.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:18.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:18.594 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:18.696 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:18.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:18.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 97581 97565 88.353 65.625 0.00896379 0.0113124 2024-09-14T04:26:18.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:18.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:18.867 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:18 smithi001 ceph-mon[112165]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:18.868 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:18 smithi001 ceph-mon[107399]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:18.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:18.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:18.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:19.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:18 smithi012 ceph-mon[101180]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:19.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:19.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:19.252 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:19.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:19.306 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:26:19.306 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:26:19.306 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:26:19.306 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:26:19.306 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:26:19.307 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:26:19.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:19.430 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:19.489 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:26:19.490 INFO:teuthology.orchestra.run.smithi001.stdout:{"fsid":"5626d550-724a-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2000,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":424,"num_osds":8,"num_up_osds":8,"osd_up_since":1726287846,"num_in_osds":6,"osd_in_since":1726287730,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":94794,"data_bytes":6211338424,"bytes_used":23564349440,"bytes_avail":552405864448,"bytes_total":575970213888,"misplaced_objects":986,"misplaced_total":284382,"misplaced_ratio":0.0034671674015936309,"recovering_objects_per_sec":60,"recovering_bytes_per_sec":3950355,"recovering_keys_per_sec":0,"num_objects_recovered":603,"num_bytes_recovered":39518208,"num_keys_recovered":0,"write_bytes_sec":67883220,"read_op_per_sec":0,"write_op_per_sec":1035},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi001.ytjgbj","status":"up:active","gid":24557}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.1:8443/","prometheus":"http://172.21.15.1:9283/"}},"servicemap":{"epoch":31,"modified":"2024-09-14T04:25:07.478423+0000","services":{}},"progress_events":{}} 2024-09-14T04:26:19.532 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:19.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:19.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:19.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:19.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 98518 98502 87.9274 58.5625 0.00988718 0.0113681 2024-09-14T04:26:19.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:19.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:19.906 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/644841247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:19.906 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1840550142' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:26:19.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/644841247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:19.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1840550142' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:26:19.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:19.985 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:19.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:20.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:20.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/644841247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:20.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1840550142' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:26:20.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:20.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:20.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:20.354 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:20.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:20.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:20.451 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:26:20.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:26:20.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:20.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:20.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:20.675 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[107399]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[112165]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:20.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:20.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 99597 99581 87.6388 67.4375 0.0121171 0.0114053 2024-09-14T04:26:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.816 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:20.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:20.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:20.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:20.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:21.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:21.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:20 smithi012 ceph-mon[101180]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 986/284382 objects misplaced (0.347%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:26:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:21.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:21.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:21.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:21.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:21.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:21.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:21.589 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:21.635 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:21.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:21.779 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:21.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:21.797 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 100685 100669 87.3659 68 0.00479765 0.0114405 2024-09-14T04:26:21.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:21.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:21.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:22.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:22.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:22.160 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:22.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:22.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:22.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:22.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:22.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:22.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:22.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:22.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:22.628 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:22 smithi001 ceph-mon[107399]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 592/294942 objects misplaced (0.201%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:26:22.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:22.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:22.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 101786 101770 87.1117 68.8125 0.0177902 0.0114747 2024-09-14T04:26:22.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:22.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:22.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:22.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:26:22.913 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:22.928 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:22.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:22 smithi001 ceph-mon[112165]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 592/294942 objects misplaced (0.201%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:26:22.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:23.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:22 smithi012 ceph-mon[101180]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 592/294942 objects misplaced (0.201%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:26:23.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:23.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:23.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:23.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:23.376 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:23.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:23.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:23.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:23.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:23.705 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:23.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 102715 102699 86.7192 58.0625 0.0131793 0.0115263 2024-09-14T04:26:23.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:23.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:23.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:23.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:23.971 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:26:23.971 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:26:24.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:24.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:24.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:24.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:24.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:24.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:24.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:24.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:24.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:24.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:24.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:24.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.709 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:24.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:24.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:24 smithi001 ceph-mon[107399]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s; 592/299964 objects misplaced (0.197%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:24.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:24 smithi001 ceph-mon[112165]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s; 592/299964 objects misplaced (0.197%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:24.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 103804 103788 86.4703 68.0625 0.00922011 0.0115594 2024-09-14T04:26:24.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:24.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:24.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:24.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:24 smithi012 ceph-mon[101180]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s; 592/299964 objects misplaced (0.197%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:25.111 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:25.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:25.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:25.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:25.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:25.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:25.518 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:25.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:25.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:25.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:25.783 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:25.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 104865 104849 86.2051 66.3125 0.00487218 0.0115952 2024-09-14T04:26:25.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:25.926 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:25.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:26.028 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:26.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:26.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:26.193 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:26.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:26.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:26.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:26.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:26.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:26.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:26.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:26.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:26.772 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:26.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:26.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 105994 105978 86.0019 70.5625 0.0134089 0.0116227 2024-09-14T04:26:26.873 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:26.910 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:26.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:26.931 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:26 smithi001 ceph-mon[112165]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 592/304839 objects misplaced (0.194%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:26.932 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:26 smithi001 ceph-mon[107399]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 592/304839 objects misplaced (0.194%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:26.932 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:26] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.51.0" 2024-09-14T04:26:26.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:26:27.012 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.065 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:27.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:27.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:26 smithi012 ceph-mon[101180]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 592/304839 objects misplaced (0.194%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:26:27.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:27.195 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:27.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:27.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:27.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:27.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.492 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:27.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:27.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:27.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:27.741 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:27.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:27.798 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 107012 106996 85.7149 63.625 0.0103585 0.0116615 2024-09-14T04:26:27.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:27.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:27.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:28.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:28.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:28.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:28.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:28.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:28.332 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:28.434 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:28.473 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:28.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:28.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:28.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:28.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:28.799 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 108081 108065 85.4751 66.8125 0.00493603 0.0116915 2024-09-14T04:26:28.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:28.884 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:28 smithi001 ceph-mon[107399]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 195/315732 objects misplaced (0.062%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:26:28.885 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:28 smithi001 ceph-mon[112165]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 195/315732 objects misplaced (0.062%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:26:28.891 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:28.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:28.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:28.993 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:29.016 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:28 smithi012 ceph-mon[101180]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 195/315732 objects misplaced (0.062%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:26:29.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:29.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:29.209 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:29.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:29.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:29.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:29.396 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:29.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:29.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:29.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:29.609 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:26:29.610 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:26:29.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:29.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:29.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:29.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:29.799 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:26:29.799468+0000 min lat: 0.00114104 max lat: 2.49143 avg lat: 0.0118122 2024-09-14T04:26:29.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:26:29.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 108343 108327 84.6109 16.375 0.00835874 0.0118122 2024-09-14T04:26:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.801 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:29.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:29.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:29.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:29.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[107399]: osdmap e425: 8 total, 8 up, 6 in 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[107399]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 195/315732 objects misplaced (0.062%); 4.9 MiB/s, 79 objects/s recovering 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[107399]: from='client.46157 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[112165]: osdmap e425: 8 total, 8 up, 6 in 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[112165]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 195/315732 objects misplaced (0.062%); 4.9 MiB/s, 79 objects/s recovering 2024-09-14T04:26:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:29 smithi001 ceph-mon[112165]: from='client.46157 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:30.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:30.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:29 smithi012 ceph-mon[101180]: osdmap e425: 8 total, 8 up, 6 in 2024-09-14T04:26:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:29 smithi012 ceph-mon[101180]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 195/315732 objects misplaced (0.062%); 4.9 MiB/s, 79 objects/s recovering 2024-09-14T04:26:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:29 smithi012 ceph-mon[101180]: from='client.46157 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:30.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:30.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:30.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:30.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:30.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:30.555 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1413,"stamp":"2024-09-14T04:26:29.510707+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6896189624,"num_objects":105244,"num_object_clones":0,"num_object_copies":315732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":195,"num_objects_unfound":0,"num_objects_dirty":105244,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":105292,"num_write_kb":6734095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5246,"num_bytes_recovered":341930235,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95606,"ondisk_log_size":95606,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":25066524,"kb_used_data":20356524,"kb_used_omap":38,"kb_used_meta":4709913,"kb_avail":537404388,"statfs":{"total":575970213888,"available":550302093312,"internally_reserved":0,"allocated":20845080576,"data_stored":20828555403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39753,"internal_metadata":4822951095},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":684851200,"num_objects":10450,"num_object_clones":0,"num_object_copies":31350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-791,"num_objects_unfound":0,"num_objects_dirty":10450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10450,"num_write_kb":668800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":791,"num_bytes_recovered":51838976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002862"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":651,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572329+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:26:09.572329+0000","last_peered":"2024-09-14T04:26:09.572329+0000","last_clean":"2024-09-14T04:26:09.572329+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:26:09.572329+0000","last_undegraded":"2024-09-14T04:26:09.572329+0000","last_fullsized":"2024-09-14T04:26:09.572329+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":157,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582229+0000","last_change":"2024-09-14T04:23:28.577777+0000","last_active":"2024-09-14T04:26:09.582229+0000","last_peered":"2024-09-14T04:26:09.582229+0000","last_clean":"2024-09-14T04:26:09.582229+0000","last_became_active":"2024-09-14T04:23:28.577155+0000","last_became_peered":"2024-09-14T04:23:28.577155+0000","last_unstale":"2024-09-14T04:26:09.582229+0000","last_undegraded":"2024-09-14T04:26:09.582229+0000","last_fullsized":"2024-09-14T04:26:09.582229+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:01.212541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":172,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616023+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:25:55.616023+0000","last_peered":"2024-09-14T04:25:55.616023+0000","last_clean":"2024-09-14T04:25:55.616023+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:25:55.616023+0000","last_undegraded":"2024-09-14T04:25:55.616023+0000","last_fullsized":"2024-09-14T04:25:55.616023+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":762,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.612433+0000","last_change":"2024-09-14T04:24:49.569053+0000","last_active":"2024-09-14T04:25:55.612433+0000","last_peered":"2024-09-14T04:25:55.612433+0000","last_clean":"2024-09-14T04:25:55.612433+0000","last_became_active":"2024-09-14T04:24:49.568690+0000","last_became_peered":"2024-09-14T04:24:49.568690+0000","last_unstale":"2024-09-14T04:25:55.612433+0000","last_undegraded":"2024-09-14T04:25:55.612433+0000","last_fullsized":"2024-09-14T04:25:55.612433+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:19:37.952706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.614755+0000","last_change":"2024-09-14T04:25:25.582150+0000","last_active":"2024-09-14T04:25:55.614755+0000","last_peered":"2024-09-14T04:25:55.614755+0000","last_clean":"2024-09-14T04:25:55.614755+0000","last_became_active":"2024-09-14T04:25:25.581683+0000","last_became_peered":"2024-09-14T04:25:25.581683+0000","last_unstale":"2024-09-14T04:25:55.614755+0000","last_undegraded":"2024-09-14T04:25:55.614755+0000","last_fullsized":"2024-09-14T04:25:55.614755+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:10.188214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":467,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.571161+0000","last_change":"2024-09-14T04:25:35.562683+0000","last_active":"2024-09-14T04:26:09.571161+0000","last_peered":"2024-09-14T04:26:09.571161+0000","last_clean":"2024-09-14T04:26:09.571161+0000","last_became_active":"2024-09-14T04:25:35.562281+0000","last_became_peered":"2024-09-14T04:25:35.562281+0000","last_unstale":"2024-09-14T04:26:09.571161+0000","last_undegraded":"2024-09-14T04:26:09.571161+0000","last_fullsized":"2024-09-14T04:26:09.571161+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:09.493757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":941,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581681+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:26:09.581681+0000","last_peered":"2024-09-14T04:26:09.581681+0000","last_clean":"2024-09-14T04:26:09.581681+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:26:09.581681+0000","last_undegraded":"2024-09-14T04:26:09.581681+0000","last_fullsized":"2024-09-14T04:26:09.581681+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":453,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575221+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:26:09.575221+0000","last_peered":"2024-09-14T04:26:09.575221+0000","last_clean":"2024-09-14T04:26:09.575221+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:26:09.575221+0000","last_undegraded":"2024-09-14T04:26:09.575221+0000","last_fullsized":"2024-09-14T04:26:09.575221+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":378,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615586+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:25:55.615586+0000","last_peered":"2024-09-14T04:25:55.615586+0000","last_clean":"2024-09-14T04:25:55.615586+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:25:55.615586+0000","last_undegraded":"2024-09-14T04:25:55.615586+0000","last_fullsized":"2024-09-14T04:25:55.615586+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":590,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616579+0000","last_change":"2024-09-14T04:24:49.572838+0000","last_active":"2024-09-14T04:25:55.616579+0000","last_peered":"2024-09-14T04:25:55.616579+0000","last_clean":"2024-09-14T04:25:55.616579+0000","last_became_active":"2024-09-14T04:24:49.572388+0000","last_became_peered":"2024-09-14T04:24:49.572388+0000","last_unstale":"2024-09-14T04:25:55.616579+0000","last_undegraded":"2024-09-14T04:25:55.616579+0000","last_fullsized":"2024-09-14T04:25:55.616579+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:22.578056+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.574242+0000","last_change":"2024-09-14T04:25:35.565538+0000","last_active":"2024-09-14T04:26:09.574242+0000","last_peered":"2024-09-14T04:26:09.574242+0000","last_clean":"2024-09-14T04:26:09.574242+0000","last_became_active":"2024-09-14T04:25:35.565210+0000","last_became_peered":"2024-09-14T04:25:35.565210+0000","last_unstale":"2024-09-14T04:26:09.574242+0000","last_undegraded":"2024-09-14T04:26:09.574242+0000","last_fullsized":"2024-09-14T04:26:09.574242+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:08.538693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":535,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576113+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:26:09.576113+0000","last_peered":"2024-09-14T04:26:09.576113+0000","last_clean":"2024-09-14T04:26:09.576113+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:26:09.576113+0000","last_undegraded":"2024-09-14T04:26:09.576113+0000","last_fullsized":"2024-09-14T04:26:09.576113+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"424'6429","reported_seq":6492,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:27.185027+0000","last_change":"2024-09-14T04:25:02.146250+0000","last_active":"2024-09-14T04:26:27.185027+0000","last_peered":"2024-09-14T04:26:27.185027+0000","last_clean":"2024-09-14T04:26:27.185027+0000","last_became_active":"2024-09-14T04:25:02.145730+0000","last_became_peered":"2024-09-14T04:25:02.145730+0000","last_unstale":"2024-09-14T04:26:27.185027+0000","last_undegraded":"2024-09-14T04:26:27.185027+0000","last_fullsized":"2024-09-14T04:26:27.185027+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6429,"log_dups_size":0,"ondisk_log_size":6429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:04.224581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421330944,"num_objects":6429,"num_object_clones":0,"num_object_copies":19287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":746,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:05.074056+0000","last_change":"2024-09-14T04:24:49.572850+0000","last_active":"2024-09-14T04:26:05.074056+0000","last_peered":"2024-09-14T04:26:05.074056+0000","last_clean":"2024-09-14T04:26:05.074056+0000","last_became_active":"2024-09-14T04:24:49.572405+0000","last_became_peered":"2024-09-14T04:24:49.572405+0000","last_unstale":"2024-09-14T04:26:05.074056+0000","last_undegraded":"2024-09-14T04:26:05.074056+0000","last_fullsized":"2024-09-14T04:26:05.074056+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:06.709118+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":140,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616453+0000","last_change":"2024-09-14T04:23:28.575634+0000","last_active":"2024-09-14T04:25:55.616453+0000","last_peered":"2024-09-14T04:25:55.616453+0000","last_clean":"2024-09-14T04:25:55.616453+0000","last_became_active":"2024-09-14T04:23:28.558391+0000","last_became_peered":"2024-09-14T04:23:28.558391+0000","last_unstale":"2024-09-14T04:25:55.616453+0000","last_undegraded":"2024-09-14T04:25:55.616453+0000","last_fullsized":"2024-09-14T04:25:55.616453+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:03.845925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.612267+0000","last_change":"2024-09-14T04:25:52.595074+0000","last_active":"2024-09-14T04:25:55.612267+0000","last_peered":"2024-09-14T04:25:55.612267+0000","last_clean":"2024-09-14T04:25:55.612267+0000","last_became_active":"2024-09-14T04:25:52.594744+0000","last_became_peered":"2024-09-14T04:25:52.594744+0000","last_unstale":"2024-09-14T04:25:55.612267+0000","last_undegraded":"2024-09-14T04:25:55.612267+0000","last_fullsized":"2024-09-14T04:25:55.612267+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:10.267321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"424'6600","reported_seq":6663,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:26.071117+0000","last_change":"2024-09-14T04:25:01.992464+0000","last_active":"2024-09-14T04:26:26.071117+0000","last_peered":"2024-09-14T04:26:26.071117+0000","last_clean":"2024-09-14T04:26:26.071117+0000","last_became_active":"2024-09-14T04:25:01.992117+0000","last_became_peered":"2024-09-14T04:25:01.992117+0000","last_unstale":"2024-09-14T04:26:26.071117+0000","last_undegraded":"2024-09-14T04:26:26.071117+0000","last_fullsized":"2024-09-14T04:26:26.071117+0000","mapping_epoch":398,"log_start":"406'1400","ondisk_log_start":"406'1400","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5200,"log_dups_size":0,"ondisk_log_size":5200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:21.289671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432537600,"num_objects":6600,"num_object_clones":0,"num_object_copies":19800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6600,"num_write_kb":422400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":994,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573730+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:26:09.573730+0000","last_peered":"2024-09-14T04:26:09.573730+0000","last_clean":"2024-09-14T04:26:09.573730+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:26:09.573730+0000","last_undegraded":"2024-09-14T04:26:09.573730+0000","last_fullsized":"2024-09-14T04:26:09.573730+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":37,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616202+0000","last_change":"2024-09-14T04:25:07.963635+0000","last_active":"2024-09-14T04:25:55.616202+0000","last_peered":"2024-09-14T04:25:55.616202+0000","last_clean":"2024-09-14T04:25:55.616202+0000","last_became_active":"2024-09-14T04:25:07.959540+0000","last_became_peered":"2024-09-14T04:25:07.959540+0000","last_unstale":"2024-09-14T04:25:55.616202+0000","last_undegraded":"2024-09-14T04:25:55.616202+0000","last_fullsized":"2024-09-14T04:25:55.616202+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:05.353685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":450,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:26:05.073889+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:26:05.073889+0000","last_peered":"2024-09-14T04:26:05.073889+0000","last_clean":"2024-09-14T04:26:05.073889+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:26:05.073889+0000","last_undegraded":"2024-09-14T04:26:05.073889+0000","last_fullsized":"2024-09-14T04:26:05.073889+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"424'6533","reported_seq":6596,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:24.532135+0000","last_change":"2024-09-14T04:25:02.146419+0000","last_active":"2024-09-14T04:26:24.532135+0000","last_peered":"2024-09-14T04:26:24.532135+0000","last_clean":"2024-09-14T04:26:24.532135+0000","last_became_active":"2024-09-14T04:25:02.145877+0000","last_became_peered":"2024-09-14T04:25:02.145877+0000","last_unstale":"2024-09-14T04:26:24.532135+0000","last_undegraded":"2024-09-14T04:26:24.532135+0000","last_fullsized":"2024-09-14T04:26:24.532135+0000","mapping_epoch":398,"log_start":"406'1000","ondisk_log_start":"406'1000","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:31.774348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428146688,"num_objects":6533,"num_object_clones":0,"num_object_copies":19599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6533,"num_write_kb":418112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":446,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.568726+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:26:09.568726+0000","last_peered":"2024-09-14T04:26:09.568726+0000","last_clean":"2024-09-14T04:26:09.568726+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:26:09.568726+0000","last_undegraded":"2024-09-14T04:26:09.568726+0000","last_fullsized":"2024-09-14T04:26:09.568726+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":31,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.613524+0000","last_change":"2024-09-14T04:25:25.584570+0000","last_active":"2024-09-14T04:25:55.613524+0000","last_peered":"2024-09-14T04:25:55.613524+0000","last_clean":"2024-09-14T04:25:55.613524+0000","last_became_active":"2024-09-14T04:25:25.581721+0000","last_became_peered":"2024-09-14T04:25:25.581721+0000","last_unstale":"2024-09-14T04:25:55.613524+0000","last_undegraded":"2024-09-14T04:25:55.613524+0000","last_fullsized":"2024-09-14T04:25:55.613524+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:59.618866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":440,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582385+0000","last_change":"2024-09-14T04:24:08.985773+0000","last_active":"2024-09-14T04:26:09.582385+0000","last_peered":"2024-09-14T04:26:09.582385+0000","last_clean":"2024-09-14T04:26:09.582385+0000","last_became_active":"2024-09-14T04:24:08.985458+0000","last_became_peered":"2024-09-14T04:24:08.985458+0000","last_unstale":"2024-09-14T04:26:09.582385+0000","last_undegraded":"2024-09-14T04:26:09.582385+0000","last_fullsized":"2024-09-14T04:26:09.582385+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:10.470781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"424'6620","reported_seq":12474,"reported_epoch":424,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:26:26.073879+0000","last_change":"2024-09-14T04:25:52.621612+0000","last_active":"2024-09-14T04:26:26.073879+0000","last_peered":"2024-09-14T04:26:26.073879+0000","last_clean":"2024-09-14T04:25:34.547607+0000","last_became_active":"2024-09-14T04:25:37.041836+0000","last_became_peered":"2024-09-14T04:25:37.041836+0000","last_unstale":"2024-09-14T04:26:26.073879+0000","last_undegraded":"2024-09-14T04:26:26.073879+0000","last_fullsized":"2024-09-14T04:25:36.555651+0000","mapping_epoch":414,"log_start":"406'800","ondisk_log_start":"406'800","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.438646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433848320,"num_objects":6620,"num_object_clones":0,"num_object_copies":19860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":195,"num_objects_unfound":0,"num_objects_dirty":6620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6620,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":189988864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":195},{"shards":"4,5","objects":6425}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":400,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616282+0000","last_change":"2024-09-14T04:25:10.210264+0000","last_active":"2024-09-14T04:25:55.616282+0000","last_peered":"2024-09-14T04:25:55.616282+0000","last_clean":"2024-09-14T04:25:55.616282+0000","last_became_active":"2024-09-14T04:25:10.209992+0000","last_became_peered":"2024-09-14T04:25:10.209992+0000","last_unstale":"2024-09-14T04:25:55.616282+0000","last_undegraded":"2024-09-14T04:25:55.616282+0000","last_fullsized":"2024-09-14T04:25:55.616282+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:50.222369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:50.222369+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":373,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572572+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:26:09.572572+0000","last_peered":"2024-09-14T04:26:09.572572+0000","last_clean":"2024-09-14T04:26:09.572572+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:26:09.572572+0000","last_undegraded":"2024-09-14T04:26:09.572572+0000","last_fullsized":"2024-09-14T04:26:09.572572+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":447,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.583179+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:26:09.583179+0000","last_peered":"2024-09-14T04:26:09.583179+0000","last_clean":"2024-09-14T04:26:09.583179+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:26:09.583179+0000","last_undegraded":"2024-09-14T04:26:09.583179+0000","last_fullsized":"2024-09-14T04:26:09.583179+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"424'6599","reported_seq":6634,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:25.082246+0000","last_change":"2024-09-14T04:25:01.992668+0000","last_active":"2024-09-14T04:26:25.082246+0000","last_peered":"2024-09-14T04:26:25.082246+0000","last_clean":"2024-09-14T04:26:25.082246+0000","last_became_active":"2024-09-14T04:25:01.992359+0000","last_became_peered":"2024-09-14T04:25:01.992359+0000","last_unstale":"2024-09-14T04:26:25.082246+0000","last_undegraded":"2024-09-14T04:26:25.082246+0000","last_fullsized":"2024-09-14T04:26:25.082246+0000","mapping_epoch":398,"log_start":"406'1500","ondisk_log_start":"406'1500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:08.434277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432472064,"num_objects":6599,"num_object_clones":0,"num_object_copies":19797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6599,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":734,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.589543+0000","last_change":"2024-09-14T04:24:10.193742+0000","last_active":"2024-09-14T04:26:09.589543+0000","last_peered":"2024-09-14T04:26:09.589543+0000","last_clean":"2024-09-14T04:26:09.589543+0000","last_became_active":"2024-09-14T04:24:10.193277+0000","last_became_peered":"2024-09-14T04:24:10.193277+0000","last_unstale":"2024-09-14T04:26:09.589543+0000","last_undegraded":"2024-09-14T04:26:09.589543+0000","last_fullsized":"2024-09-14T04:26:09.589543+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:08.270894+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":451,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.580235+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:26:09.580235+0000","last_peered":"2024-09-14T04:26:09.580235+0000","last_clean":"2024-09-14T04:26:09.580235+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:26:09.580235+0000","last_undegraded":"2024-09-14T04:26:09.580235+0000","last_fullsized":"2024-09-14T04:26:09.580235+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":425,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615964+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:25:55.615964+0000","last_peered":"2024-09-14T04:25:55.615964+0000","last_clean":"2024-09-14T04:25:55.615964+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:25:55.615964+0000","last_undegraded":"2024-09-14T04:25:55.615964+0000","last_fullsized":"2024-09-14T04:25:55.615964+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"424'6702","reported_seq":6762,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:27.220061+0000","last_change":"2024-09-14T04:25:01.995435+0000","last_active":"2024-09-14T04:26:27.220061+0000","last_peered":"2024-09-14T04:26:27.220061+0000","last_clean":"2024-09-14T04:26:27.220061+0000","last_became_active":"2024-09-14T04:25:01.995177+0000","last_became_peered":"2024-09-14T04:25:01.995177+0000","last_unstale":"2024-09-14T04:26:27.220061+0000","last_undegraded":"2024-09-14T04:26:27.220061+0000","last_fullsized":"2024-09-14T04:26:27.220061+0000","mapping_epoch":398,"log_start":"405'100","ondisk_log_start":"405'100","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:02.315254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439222272,"num_objects":6702,"num_object_clones":0,"num_object_copies":20106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6702,"num_write_kb":428928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":658,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.612517+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:25:55.612517+0000","last_peered":"2024-09-14T04:25:55.612517+0000","last_clean":"2024-09-14T04:25:55.612517+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:25:55.612517+0000","last_undegraded":"2024-09-14T04:25:55.612517+0000","last_fullsized":"2024-09-14T04:25:55.612517+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":529,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.571539+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:26:09.571539+0000","last_peered":"2024-09-14T04:26:09.571539+0000","last_clean":"2024-09-14T04:26:09.571539+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:26:09.571539+0000","last_undegraded":"2024-09-14T04:26:09.571539+0000","last_fullsized":"2024-09-14T04:26:09.571539+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":33,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616545+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:25:55.616545+0000","last_peered":"2024-09-14T04:25:55.616545+0000","last_clean":"2024-09-14T04:25:55.616545+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:25:55.616545+0000","last_undegraded":"2024-09-14T04:25:55.616545+0000","last_fullsized":"2024-09-14T04:25:55.616545+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"424'6529","reported_seq":11141,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:25.071464+0000","last_change":"2024-09-14T04:25:44.643768+0000","last_active":"2024-09-14T04:26:25.071464+0000","last_peered":"2024-09-14T04:26:25.071464+0000","last_clean":"2024-09-14T04:26:25.071464+0000","last_became_active":"2024-09-14T04:25:44.638986+0000","last_became_peered":"2024-09-14T04:25:44.638986+0000","last_unstale":"2024-09-14T04:26:25.071464+0000","last_undegraded":"2024-09-14T04:26:25.071464+0000","last_fullsized":"2024-09-14T04:26:25.071464+0000","mapping_epoch":419,"log_start":"406'1400","ondisk_log_start":"406'1400","created":398,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5129,"log_dups_size":0,"ondisk_log_size":5129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:07.177695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427884544,"num_objects":6529,"num_object_clones":0,"num_object_copies":19587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6529,"num_write_kb":417856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":921,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581333+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:26:09.581333+0000","last_peered":"2024-09-14T04:26:09.581333+0000","last_clean":"2024-09-14T04:26:09.581333+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:26:09.581333+0000","last_undegraded":"2024-09-14T04:26:09.581333+0000","last_fullsized":"2024-09-14T04:26:09.581333+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":33,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615229+0000","last_change":"2024-09-14T04:25:07.967648+0000","last_active":"2024-09-14T04:25:55.615229+0000","last_peered":"2024-09-14T04:25:55.615229+0000","last_clean":"2024-09-14T04:25:55.615229+0000","last_became_active":"2024-09-14T04:25:07.967284+0000","last_became_peered":"2024-09-14T04:25:07.967284+0000","last_unstale":"2024-09-14T04:25:55.615229+0000","last_undegraded":"2024-09-14T04:25:55.615229+0000","last_fullsized":"2024-09-14T04:25:55.615229+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:29.967190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":431,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616471+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:25:55.616471+0000","last_peered":"2024-09-14T04:25:55.616471+0000","last_clean":"2024-09-14T04:25:55.616471+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:25:55.616471+0000","last_undegraded":"2024-09-14T04:25:55.616471+0000","last_fullsized":"2024-09-14T04:25:55.616471+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"424'6512","reported_seq":6547,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:25.087990+0000","last_change":"2024-09-14T04:25:02.146105+0000","last_active":"2024-09-14T04:26:25.087990+0000","last_peered":"2024-09-14T04:26:25.087990+0000","last_clean":"2024-09-14T04:26:25.087990+0000","last_became_active":"2024-09-14T04:25:02.145736+0000","last_became_peered":"2024-09-14T04:25:02.145736+0000","last_unstale":"2024-09-14T04:26:25.087990+0000","last_undegraded":"2024-09-14T04:26:25.087990+0000","last_fullsized":"2024-09-14T04:26:25.087990+0000","mapping_epoch":398,"log_start":"406'1400","ondisk_log_start":"406'1400","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.907164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426770432,"num_objects":6512,"num_object_clones":0,"num_object_copies":19536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":629,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573315+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:26:09.573315+0000","last_peered":"2024-09-14T04:26:09.573315+0000","last_clean":"2024-09-14T04:26:09.573315+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:26:09.573315+0000","last_undegraded":"2024-09-14T04:26:09.573315+0000","last_fullsized":"2024-09-14T04:26:09.573315+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572802+0000","last_change":"2024-09-14T04:25:35.561629+0000","last_active":"2024-09-14T04:26:09.572802+0000","last_peered":"2024-09-14T04:26:09.572802+0000","last_clean":"2024-09-14T04:26:09.572802+0000","last_became_active":"2024-09-14T04:25:35.561305+0000","last_became_peered":"2024-09-14T04:25:35.561305+0000","last_unstale":"2024-09-14T04:26:09.572802+0000","last_undegraded":"2024-09-14T04:26:09.572802+0000","last_fullsized":"2024-09-14T04:26:09.572802+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:39.574054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":690,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582837+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:26:09.582837+0000","last_peered":"2024-09-14T04:26:09.582837+0000","last_clean":"2024-09-14T04:26:09.582837+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:26:09.582837+0000","last_undegraded":"2024-09-14T04:26:09.582837+0000","last_fullsized":"2024-09-14T04:26:09.582837+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"424'6664","reported_seq":6726,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:26.091801+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:26:26.091801+0000","last_peered":"2024-09-14T04:26:26.091801+0000","last_clean":"2024-09-14T04:26:26.091801+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:26:26.091801+0000","last_undegraded":"2024-09-14T04:26:26.091801+0000","last_fullsized":"2024-09-14T04:26:26.091801+0000","mapping_epoch":398,"log_start":"406'1400","ondisk_log_start":"406'1400","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:09.181654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436731904,"num_objects":6664,"num_object_clones":0,"num_object_copies":19992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6664,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":790,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576254+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:26:09.576254+0000","last_peered":"2024-09-14T04:26:09.576254+0000","last_clean":"2024-09-14T04:26:09.576254+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:26:09.576254+0000","last_undegraded":"2024-09-14T04:26:09.576254+0000","last_fullsized":"2024-09-14T04:26:09.576254+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:17.530093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":363,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572438+0000","last_change":"2024-09-14T04:24:08.979706+0000","last_active":"2024-09-14T04:26:09.572438+0000","last_peered":"2024-09-14T04:26:09.572438+0000","last_clean":"2024-09-14T04:26:09.572438+0000","last_became_active":"2024-09-14T04:24:08.979397+0000","last_became_peered":"2024-09-14T04:24:08.979397+0000","last_unstale":"2024-09-14T04:26:09.572438+0000","last_undegraded":"2024-09-14T04:26:09.572438+0000","last_fullsized":"2024-09-14T04:26:09.572438+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:12.232467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576917+0000","last_change":"2024-09-14T04:25:52.594667+0000","last_active":"2024-09-14T04:26:09.576917+0000","last_peered":"2024-09-14T04:26:09.576917+0000","last_clean":"2024-09-14T04:26:09.576917+0000","last_became_active":"2024-09-14T04:25:52.594400+0000","last_became_peered":"2024-09-14T04:25:52.594400+0000","last_unstale":"2024-09-14T04:26:09.576917+0000","last_undegraded":"2024-09-14T04:26:09.576917+0000","last_fullsized":"2024-09-14T04:26:09.576917+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:12.710417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":228,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616312+0000","last_change":"2024-09-14T04:25:35.560355+0000","last_active":"2024-09-14T04:25:55.616312+0000","last_peered":"2024-09-14T04:25:55.616312+0000","last_clean":"2024-09-14T04:25:55.616312+0000","last_became_active":"2024-09-14T04:25:35.559975+0000","last_became_peered":"2024-09-14T04:25:35.559975+0000","last_unstale":"2024-09-14T04:25:55.616312+0000","last_undegraded":"2024-09-14T04:25:55.616312+0000","last_fullsized":"2024-09-14T04:25:55.616312+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:58.033714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"424'6658","reported_seq":6727,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:27.189224+0000","last_change":"2024-09-14T04:25:07.966414+0000","last_active":"2024-09-14T04:26:27.189224+0000","last_peered":"2024-09-14T04:26:27.189224+0000","last_clean":"2024-09-14T04:26:27.189224+0000","last_became_active":"2024-09-14T04:25:07.966014+0000","last_became_peered":"2024-09-14T04:25:07.966014+0000","last_unstale":"2024-09-14T04:26:27.189224+0000","last_undegraded":"2024-09-14T04:26:27.189224+0000","last_fullsized":"2024-09-14T04:26:27.189224+0000","mapping_epoch":401,"log_start":"405'100","ondisk_log_start":"405'100","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:58.062778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436338688,"num_objects":6658,"num_object_clones":0,"num_object_copies":19974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6658,"num_write_kb":426112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":912,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581444+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:26:09.581444+0000","last_peered":"2024-09-14T04:26:09.581444+0000","last_clean":"2024-09-14T04:26:09.581444+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:26:09.581444+0000","last_undegraded":"2024-09-14T04:26:09.581444+0000","last_fullsized":"2024-09-14T04:26:09.581444+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":391,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581823+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:26:09.581823+0000","last_peered":"2024-09-14T04:26:09.581823+0000","last_clean":"2024-09-14T04:26:09.581823+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:26:09.581823+0000","last_undegraded":"2024-09-14T04:26:09.581823+0000","last_fullsized":"2024-09-14T04:26:09.581823+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":696,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572182+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:26:09.572182+0000","last_peered":"2024-09-14T04:26:09.572182+0000","last_clean":"2024-09-14T04:26:09.572182+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:26:09.572182+0000","last_undegraded":"2024-09-14T04:26:09.572182+0000","last_fullsized":"2024-09-14T04:26:09.572182+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575294+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:26:09.575294+0000","last_peered":"2024-09-14T04:26:09.575294+0000","last_clean":"2024-09-14T04:26:09.575294+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:26:09.575294+0000","last_undegraded":"2024-09-14T04:26:09.575294+0000","last_fullsized":"2024-09-14T04:26:09.575294+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"424'6469","reported_seq":6504,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:26.047167+0000","last_change":"2024-09-14T04:25:01.992854+0000","last_active":"2024-09-14T04:26:26.047167+0000","last_peered":"2024-09-14T04:26:26.047167+0000","last_clean":"2024-09-14T04:26:26.047167+0000","last_became_active":"2024-09-14T04:25:01.992514+0000","last_became_peered":"2024-09-14T04:25:01.992514+0000","last_unstale":"2024-09-14T04:26:26.047167+0000","last_undegraded":"2024-09-14T04:26:26.047167+0000","last_fullsized":"2024-09-14T04:26:26.047167+0000","mapping_epoch":398,"log_start":"406'600","ondisk_log_start":"406'600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:59.789126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423952384,"num_objects":6469,"num_object_clones":0,"num_object_copies":19407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6469,"num_write_kb":414016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":665,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.571366+0000","last_change":"2024-09-14T04:25:10.213700+0000","last_active":"2024-09-14T04:26:09.571366+0000","last_peered":"2024-09-14T04:26:09.571366+0000","last_clean":"2024-09-14T04:26:09.571366+0000","last_became_active":"2024-09-14T04:25:10.213291+0000","last_became_peered":"2024-09-14T04:25:10.213291+0000","last_unstale":"2024-09-14T04:26:09.571366+0000","last_undegraded":"2024-09-14T04:26:09.571366+0000","last_fullsized":"2024-09-14T04:26:09.571366+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:58.394641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":664,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.589754+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:26:09.589754+0000","last_peered":"2024-09-14T04:26:09.589754+0000","last_clean":"2024-09-14T04:26:09.589754+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:26:09.589754+0000","last_undegraded":"2024-09-14T04:26:09.589754+0000","last_fullsized":"2024-09-14T04:26:09.589754+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":695,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581941+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:26:09.581941+0000","last_peered":"2024-09-14T04:26:09.581941+0000","last_clean":"2024-09-14T04:26:09.581941+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:26:09.581941+0000","last_undegraded":"2024-09-14T04:26:09.581941+0000","last_fullsized":"2024-09-14T04:26:09.581941+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":163,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582407+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:26:09.582407+0000","last_peered":"2024-09-14T04:26:09.582407+0000","last_clean":"2024-09-14T04:26:09.582407+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:26:09.582407+0000","last_undegraded":"2024-09-14T04:26:09.582407+0000","last_fullsized":"2024-09-14T04:26:09.582407+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"424'6599","reported_seq":6634,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:25.079531+0000","last_change":"2024-09-14T04:25:02.146113+0000","last_active":"2024-09-14T04:26:25.079531+0000","last_peered":"2024-09-14T04:26:25.079531+0000","last_clean":"2024-09-14T04:26:25.079531+0000","last_became_active":"2024-09-14T04:25:02.145762+0000","last_became_peered":"2024-09-14T04:25:02.145762+0000","last_unstale":"2024-09-14T04:26:25.079531+0000","last_undegraded":"2024-09-14T04:26:25.079531+0000","last_fullsized":"2024-09-14T04:26:25.079531+0000","mapping_epoch":398,"log_start":"406'1500","ondisk_log_start":"406'1500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:33.000554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432472064,"num_objects":6599,"num_object_clones":0,"num_object_copies":19797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6599,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":652,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.567599+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:26:09.567599+0000","last_peered":"2024-09-14T04:26:09.567599+0000","last_clean":"2024-09-14T04:26:09.567599+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:26:09.567599+0000","last_undegraded":"2024-09-14T04:26:09.567599+0000","last_fullsized":"2024-09-14T04:26:09.567599+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":360,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573233+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:26:09.573233+0000","last_peered":"2024-09-14T04:26:09.573233+0000","last_clean":"2024-09-14T04:26:09.573233+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:26:09.573233+0000","last_undegraded":"2024-09-14T04:26:09.573233+0000","last_fullsized":"2024-09-14T04:26:09.573233+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":443,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575797+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:26:09.575797+0000","last_peered":"2024-09-14T04:26:09.575797+0000","last_clean":"2024-09-14T04:26:09.575797+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:26:09.575797+0000","last_undegraded":"2024-09-14T04:26:09.575797+0000","last_fullsized":"2024-09-14T04:26:09.575797+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":380,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572647+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:26:09.572647+0000","last_peered":"2024-09-14T04:26:09.572647+0000","last_clean":"2024-09-14T04:26:09.572647+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:26:09.572647+0000","last_undegraded":"2024-09-14T04:26:09.572647+0000","last_fullsized":"2024-09-14T04:26:09.572647+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"424'6694","reported_seq":6729,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:26.056871+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:26:26.056871+0000","last_peered":"2024-09-14T04:26:26.056871+0000","last_clean":"2024-09-14T04:26:26.056871+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:26:26.056871+0000","last_undegraded":"2024-09-14T04:26:26.056871+0000","last_fullsized":"2024-09-14T04:26:26.056871+0000","mapping_epoch":398,"log_start":"406'900","ondisk_log_start":"406'900","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438697984,"num_objects":6694,"num_object_clones":0,"num_object_copies":20082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6694,"num_write_kb":428416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":684,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.574137+0000","last_change":"2024-09-14T04:25:38.633170+0000","last_active":"2024-09-14T04:26:09.574137+0000","last_peered":"2024-09-14T04:26:09.574137+0000","last_clean":"2024-09-14T04:26:09.574137+0000","last_became_active":"2024-09-14T04:25:38.632816+0000","last_became_peered":"2024-09-14T04:25:38.632816+0000","last_unstale":"2024-09-14T04:26:09.574137+0000","last_undegraded":"2024-09-14T04:26:09.574137+0000","last_fullsized":"2024-09-14T04:26:09.574137+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:55:39.218376+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":389,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572616+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:26:09.572616+0000","last_peered":"2024-09-14T04:26:09.572616+0000","last_clean":"2024-09-14T04:26:09.572616+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:26:09.572616+0000","last_undegraded":"2024-09-14T04:26:09.572616+0000","last_fullsized":"2024-09-14T04:26:09.572616+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":752,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582852+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:26:09.582852+0000","last_peered":"2024-09-14T04:26:09.582852+0000","last_clean":"2024-09-14T04:26:09.582852+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:26:09.582852+0000","last_undegraded":"2024-09-14T04:26:09.582852+0000","last_fullsized":"2024-09-14T04:26:09.582852+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":178,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582199+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:26:09.582199+0000","last_peered":"2024-09-14T04:26:09.582199+0000","last_clean":"2024-09-14T04:26:09.582199+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:26:09.582199+0000","last_undegraded":"2024-09-14T04:26:09.582199+0000","last_fullsized":"2024-09-14T04:26:09.582199+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"424'6558","reported_seq":6621,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:27.206289+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:26:27.206289+0000","last_peered":"2024-09-14T04:26:27.206289+0000","last_clean":"2024-09-14T04:26:27.206289+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:26:27.206289+0000","last_undegraded":"2024-09-14T04:26:27.206289+0000","last_fullsized":"2024-09-14T04:26:27.206289+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:13.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429785088,"num_objects":6558,"num_object_clones":0,"num_object_copies":19674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6558,"num_write_kb":419712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":702,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.574900+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:26:09.574900+0000","last_peered":"2024-09-14T04:26:09.574900+0000","last_clean":"2024-09-14T04:26:09.574900+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:26:09.574900+0000","last_undegraded":"2024-09-14T04:26:09.574900+0000","last_fullsized":"2024-09-14T04:26:09.574900+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":698,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582737+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:26:09.582737+0000","last_peered":"2024-09-14T04:26:09.582737+0000","last_clean":"2024-09-14T04:26:09.582737+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:26:09.582737+0000","last_undegraded":"2024-09-14T04:26:09.582737+0000","last_fullsized":"2024-09-14T04:26:09.582737+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":430,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573769+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:26:09.573769+0000","last_peered":"2024-09-14T04:26:09.573769+0000","last_clean":"2024-09-14T04:26:09.573769+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:26:09.573769+0000","last_undegraded":"2024-09-14T04:26:09.573769+0000","last_fullsized":"2024-09-14T04:26:09.573769+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":463,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.580120+0000","last_change":"2024-09-14T04:24:08.989110+0000","last_active":"2024-09-14T04:26:09.580120+0000","last_peered":"2024-09-14T04:26:09.580120+0000","last_clean":"2024-09-14T04:26:09.580120+0000","last_became_active":"2024-09-14T04:24:08.987440+0000","last_became_peered":"2024-09-14T04:24:08.987440+0000","last_unstale":"2024-09-14T04:26:09.580120+0000","last_undegraded":"2024-09-14T04:26:09.580120+0000","last_fullsized":"2024-09-14T04:26:09.580120+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:17:32.812292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"424'6498","reported_seq":6533,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:26.058207+0000","last_change":"2024-09-14T04:25:01.993057+0000","last_active":"2024-09-14T04:26:26.058207+0000","last_peered":"2024-09-14T04:26:26.058207+0000","last_clean":"2024-09-14T04:26:26.058207+0000","last_became_active":"2024-09-14T04:25:01.992668+0000","last_became_peered":"2024-09-14T04:25:01.992668+0000","last_unstale":"2024-09-14T04:26:26.058207+0000","last_undegraded":"2024-09-14T04:26:26.058207+0000","last_fullsized":"2024-09-14T04:26:26.058207+0000","mapping_epoch":398,"log_start":"406'700","ondisk_log_start":"406'700","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:38.544577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425852928,"num_objects":6498,"num_object_clones":0,"num_object_copies":19494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6498,"num_write_kb":415872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":729,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616083+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:25:55.616083+0000","last_peered":"2024-09-14T04:25:55.616083+0000","last_clean":"2024-09-14T04:25:55.616083+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:25:55.616083+0000","last_undegraded":"2024-09-14T04:25:55.616083+0000","last_fullsized":"2024-09-14T04:25:55.616083+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":352,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575566+0000","last_change":"2024-09-14T04:24:08.999455+0000","last_active":"2024-09-14T04:26:09.575566+0000","last_peered":"2024-09-14T04:26:09.575566+0000","last_clean":"2024-09-14T04:26:09.575566+0000","last_became_active":"2024-09-14T04:24:08.994093+0000","last_became_peered":"2024-09-14T04:24:08.994093+0000","last_unstale":"2024-09-14T04:26:09.575566+0000","last_undegraded":"2024-09-14T04:26:09.575566+0000","last_fullsized":"2024-09-14T04:26:09.575566+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:29:31.948482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":684,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582617+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:26:09.582617+0000","last_peered":"2024-09-14T04:26:09.582617+0000","last_clean":"2024-09-14T04:26:09.582617+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:26:09.582617+0000","last_undegraded":"2024-09-14T04:26:09.582617+0000","last_fullsized":"2024-09-14T04:26:09.582617+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":383,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576022+0000","last_change":"2024-09-14T04:25:35.562908+0000","last_active":"2024-09-14T04:26:09.576022+0000","last_peered":"2024-09-14T04:26:09.576022+0000","last_clean":"2024-09-14T04:26:09.576022+0000","last_became_active":"2024-09-14T04:25:35.562023+0000","last_became_peered":"2024-09-14T04:25:35.562023+0000","last_unstale":"2024-09-14T04:26:09.576022+0000","last_undegraded":"2024-09-14T04:26:09.576022+0000","last_fullsized":"2024-09-14T04:26:09.576022+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:55.656734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.8","version":"424'6554","reported_seq":6617,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:27.223761+0000","last_change":"2024-09-14T04:25:01.996105+0000","last_active":"2024-09-14T04:26:27.223761+0000","last_peered":"2024-09-14T04:26:27.223761+0000","last_clean":"2024-09-14T04:26:27.223761+0000","last_became_active":"2024-09-14T04:25:01.995811+0000","last_became_peered":"2024-09-14T04:25:01.995811+0000","last_unstale":"2024-09-14T04:26:27.223761+0000","last_undegraded":"2024-09-14T04:26:27.223761+0000","last_fullsized":"2024-09-14T04:26:27.223761+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:04.759721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429522944,"num_objects":6554,"num_object_clones":0,"num_object_copies":19662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":973,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573643+0000","last_change":"2024-09-14T04:25:28.605221+0000","last_active":"2024-09-14T04:26:09.573643+0000","last_peered":"2024-09-14T04:26:09.573643+0000","last_clean":"2024-09-14T04:26:09.573643+0000","last_became_active":"2024-09-14T04:25:28.604819+0000","last_became_peered":"2024-09-14T04:25:28.604819+0000","last_unstale":"2024-09-14T04:26:09.573643+0000","last_undegraded":"2024-09-14T04:26:09.573643+0000","last_fullsized":"2024-09-14T04:26:09.573643+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:21.033138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":462,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575135+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:26:09.575135+0000","last_peered":"2024-09-14T04:26:09.575135+0000","last_clean":"2024-09-14T04:26:09.575135+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:26:09.575135+0000","last_undegraded":"2024-09-14T04:26:09.575135+0000","last_fullsized":"2024-09-14T04:26:09.575135+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":528,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.577016+0000","last_change":"2024-09-14T04:23:27.596032+0000","last_active":"2024-09-14T04:26:09.577016+0000","last_peered":"2024-09-14T04:26:09.577016+0000","last_clean":"2024-09-14T04:26:09.577016+0000","last_became_active":"2024-09-14T04:23:27.595521+0000","last_became_peered":"2024-09-14T04:23:27.595521+0000","last_unstale":"2024-09-14T04:26:09.577016+0000","last_undegraded":"2024-09-14T04:26:09.577016+0000","last_fullsized":"2024-09-14T04:26:09.577016+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:14.145527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":409,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.570310+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:26:09.570310+0000","last_peered":"2024-09-14T04:26:09.570310+0000","last_clean":"2024-09-14T04:26:09.570310+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:26:09.570310+0000","last_undegraded":"2024-09-14T04:26:09.570310+0000","last_fullsized":"2024-09-14T04:26:09.570310+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":445,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616166+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:25:55.616166+0000","last_peered":"2024-09-14T04:25:55.616166+0000","last_clean":"2024-09-14T04:25:55.616166+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:25:55.616166+0000","last_undegraded":"2024-09-14T04:25:55.616166+0000","last_fullsized":"2024-09-14T04:25:55.616166+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":444,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.616024+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:25:55.616024+0000","last_peered":"2024-09-14T04:25:55.616024+0000","last_clean":"2024-09-14T04:25:55.616024+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:25:55.616024+0000","last_undegraded":"2024-09-14T04:25:55.616024+0000","last_fullsized":"2024-09-14T04:25:55.616024+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":149,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.582352+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:26:09.582352+0000","last_peered":"2024-09-14T04:26:09.582352+0000","last_clean":"2024-09-14T04:26:09.582352+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:26:09.582352+0000","last_undegraded":"2024-09-14T04:26:09.582352+0000","last_fullsized":"2024-09-14T04:26:09.582352+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":367,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575188+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:26:09.575188+0000","last_peered":"2024-09-14T04:26:09.575188+0000","last_clean":"2024-09-14T04:26:09.575188+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:26:09.575188+0000","last_undegraded":"2024-09-14T04:26:09.575188+0000","last_fullsized":"2024-09-14T04:26:09.575188+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":908,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.584094+0000","last_change":"2024-09-14T04:25:10.214026+0000","last_active":"2024-09-14T04:26:09.584094+0000","last_peered":"2024-09-14T04:26:09.584094+0000","last_clean":"2024-09-14T04:26:09.584094+0000","last_became_active":"2024-09-14T04:25:10.212511+0000","last_became_peered":"2024-09-14T04:25:10.212511+0000","last_unstale":"2024-09-14T04:26:09.584094+0000","last_undegraded":"2024-09-14T04:26:09.584094+0000","last_fullsized":"2024-09-14T04:26:09.584094+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:20.633170+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.613760+0000","last_change":"2024-09-14T04:25:35.559702+0000","last_active":"2024-09-14T04:25:55.613760+0000","last_peered":"2024-09-14T04:25:55.613760+0000","last_clean":"2024-09-14T04:25:55.613760+0000","last_became_active":"2024-09-14T04:25:35.559395+0000","last_became_peered":"2024-09-14T04:25:35.559395+0000","last_unstale":"2024-09-14T04:25:55.613760+0000","last_undegraded":"2024-09-14T04:25:55.613760+0000","last_fullsized":"2024-09-14T04:25:55.613760+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:51.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":305,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.612610+0000","last_change":"2024-09-14T04:24:09.122059+0000","last_active":"2024-09-14T04:25:55.612610+0000","last_peered":"2024-09-14T04:25:55.612610+0000","last_clean":"2024-09-14T04:25:55.612610+0000","last_became_active":"2024-09-14T04:24:09.121824+0000","last_became_peered":"2024-09-14T04:24:09.121824+0000","last_unstale":"2024-09-14T04:25:55.612610+0000","last_undegraded":"2024-09-14T04:25:55.612610+0000","last_fullsized":"2024-09-14T04:25:55.612610+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:36.179815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":457,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575759+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:26:09.575759+0000","last_peered":"2024-09-14T04:26:09.575759+0000","last_clean":"2024-09-14T04:26:09.575759+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:26:09.575759+0000","last_undegraded":"2024-09-14T04:26:09.575759+0000","last_fullsized":"2024-09-14T04:26:09.575759+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":561,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.573103+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:26:09.573103+0000","last_peered":"2024-09-14T04:26:09.573103+0000","last_clean":"2024-09-14T04:26:09.573103+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:26:09.573103+0000","last_undegraded":"2024-09-14T04:26:09.573103+0000","last_fullsized":"2024-09-14T04:26:09.573103+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.572892+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:26:09.572892+0000","last_peered":"2024-09-14T04:26:09.572892+0000","last_clean":"2024-09-14T04:26:09.572892+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:26:09.572892+0000","last_undegraded":"2024-09-14T04:26:09.572892+0000","last_fullsized":"2024-09-14T04:26:09.572892+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:56.035119+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:25:56.035119+0000","last_peered":"2024-09-14T04:25:56.035119+0000","last_clean":"2024-09-14T04:25:56.035119+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:25:56.035119+0000","last_undegraded":"2024-09-14T04:25:56.035119+0000","last_fullsized":"2024-09-14T04:25:56.035119+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":473,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.575058+0000","last_change":"2024-09-14T04:25:35.564324+0000","last_active":"2024-09-14T04:26:09.575058+0000","last_peered":"2024-09-14T04:26:09.575058+0000","last_clean":"2024-09-14T04:26:09.575058+0000","last_became_active":"2024-09-14T04:25:35.562918+0000","last_became_peered":"2024-09-14T04:25:35.562918+0000","last_unstale":"2024-09-14T04:26:09.575058+0000","last_undegraded":"2024-09-14T04:26:09.575058+0000","last_fullsized":"2024-09-14T04:26:09.575058+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:16.172185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":721,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.581179+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:26:09.581179+0000","last_peered":"2024-09-14T04:26:09.581179+0000","last_clean":"2024-09-14T04:26:09.581179+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:26:09.581179+0000","last_undegraded":"2024-09-14T04:26:09.581179+0000","last_fullsized":"2024-09-14T04:26:09.581179+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":273,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.614688+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:25:55.614688+0000","last_peered":"2024-09-14T04:25:55.614688+0000","last_clean":"2024-09-14T04:25:55.614688+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:25:55.614688+0000","last_undegraded":"2024-09-14T04:25:55.614688+0000","last_fullsized":"2024-09-14T04:25:55.614688+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":13,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615337+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:25:55.615337+0000","last_peered":"2024-09-14T04:25:55.615337+0000","last_clean":"2024-09-14T04:25:55.615337+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:25:55.615337+0000","last_undegraded":"2024-09-14T04:25:55.615337+0000","last_fullsized":"2024-09-14T04:25:55.615337+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.584235+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:26:09.584235+0000","last_peered":"2024-09-14T04:26:09.584235+0000","last_clean":"2024-09-14T04:26:09.584235+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:26:09.584235+0000","last_undegraded":"2024-09-14T04:26:09.584235+0000","last_fullsized":"2024-09-14T04:26:09.584235+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":471,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.579715+0000","last_change":"2024-09-14T04:23:27.573928+0000","last_active":"2024-09-14T04:26:09.579715+0000","last_peered":"2024-09-14T04:26:09.579715+0000","last_clean":"2024-09-14T04:26:09.579715+0000","last_became_active":"2024-09-14T04:23:27.573557+0000","last_became_peered":"2024-09-14T04:23:27.573557+0000","last_unstale":"2024-09-14T04:26:09.579715+0000","last_undegraded":"2024-09-14T04:26:09.579715+0000","last_fullsized":"2024-09-14T04:26:09.579715+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:07.509590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":33,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615737+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:25:55.615737+0000","last_peered":"2024-09-14T04:25:55.615737+0000","last_clean":"2024-09-14T04:25:55.615737+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:25:55.615737+0000","last_undegraded":"2024-09-14T04:25:55.615737+0000","last_fullsized":"2024-09-14T04:25:55.615737+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":371,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.614801+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:25:55.614801+0000","last_peered":"2024-09-14T04:25:55.614801+0000","last_clean":"2024-09-14T04:25:55.614801+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:25:55.614801+0000","last_undegraded":"2024-09-14T04:25:55.614801+0000","last_fullsized":"2024-09-14T04:25:55.614801+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.615582+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:25:55.615582+0000","last_peered":"2024-09-14T04:25:55.615582+0000","last_clean":"2024-09-14T04:25:55.615582+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:25:55.615582+0000","last_undegraded":"2024-09-14T04:25:55.615582+0000","last_fullsized":"2024-09-14T04:25:55.615582+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":380,"reported_epoch":423,"state":"active+clean","last_fresh":"2024-09-14T04:25:55.614531+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:25:55.614531+0000","last_peered":"2024-09-14T04:25:55.614531+0000","last_clean":"2024-09-14T04:25:55.614531+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:25:55.614531+0000","last_undegraded":"2024-09-14T04:25:55.614531+0000","last_fullsized":"2024-09-14T04:25:55.614531+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":610,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.589638+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:26:09.589638+0000","last_peered":"2024-09-14T04:26:09.589638+0000","last_clean":"2024-09-14T04:26:09.589638+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:26:09.589638+0000","last_undegraded":"2024-09-14T04:26:09.589638+0000","last_fullsized":"2024-09-14T04:26:09.589638+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":282,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576616+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:26:09.576616+0000","last_peered":"2024-09-14T04:26:09.576616+0000","last_clean":"2024-09-14T04:26:09.576616+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:26:09.576616+0000","last_undegraded":"2024-09-14T04:26:09.576616+0000","last_fullsized":"2024-09-14T04:26:09.576616+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":406,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.568017+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:26:09.568017+0000","last_peered":"2024-09-14T04:26:09.568017+0000","last_clean":"2024-09-14T04:26:09.568017+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:26:09.568017+0000","last_undegraded":"2024-09-14T04:26:09.568017+0000","last_fullsized":"2024-09-14T04:26:09.568017+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":505,"reported_epoch":424,"state":"active+clean","last_fresh":"2024-09-14T04:26:09.576016+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:26:09.576016+0000","last_peered":"2024-09-14T04:26:09.576016+0000","last_clean":"2024-09-14T04:26:09.576016+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:26:09.576016+0000","last_undegraded":"2024-09-14T04:26:09.576016+0000","last_fullsized":"2024-09-14T04:26:09.576016+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":6895566848,"num_objects":105218,"num_object_clones":0,"num_object_copies":315654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":195,"num_objects_unfound":0,"num_objects_dirty":105218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105218,"num_write_kb":6733952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5179,"num_bytes_recovered":339410944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20922826752,"data_stored":20922826752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92418,"ondisk_log_size":92418,"up":48,"acting":47,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":12322,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2506969,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201830,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463468,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834141,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4803540,"kb_used_data":4609612,"kb_used_omap":5,"kb_used_meta":193914,"kb_avail":88941612,"statfs":{"total":95995035648,"available":91076210688,"internally_reserved":0,"allocated":4720242688,"data_stored":4717492833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":198567997},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986740,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5522592,"kb_used_data":4156056,"kb_used_omap":1,"kb_used_meta":1366526,"kb_avail":88222560,"statfs":{"total":95995035648,"available":90339901440,"internally_reserved":0,"allocated":4255801344,"data_stored":4253042966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1399323078},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748769,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3578772,"kb_used_data":2598540,"kb_used_omap":4,"kb_used_meta":980219,"kb_avail":90166380,"statfs":{"total":95995035648,"available":92330373120,"internally_reserved":0,"allocated":2660904960,"data_stored":2658154887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1003744812},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286990,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4309152,"kb_used_data":3806552,"kb_used_omap":10,"kb_used_meta":502581,"kb_avail":89436000,"statfs":{"total":95995035648,"available":91582464000,"internally_reserved":0,"allocated":3897909248,"data_stored":3895152099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":514643543},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468312,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3526860,"kb_used_data":2517572,"kb_used_omap":5,"kb_used_meta":1009274,"kb_avail":90218292,"statfs":{"total":95995035648,"available":92383531008,"internally_reserved":0,"allocated":2577993728,"data_stored":2575249764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1033496927},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342427,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3325608,"kb_used_data":2668192,"kb_used_omap":10,"kb_used_meta":657397,"kb_avail":90419544,"statfs":{"total":95995035648,"available":92589613056,"internally_reserved":0,"allocated":2732228608,"data_stored":2729462854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":673174738},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2747858944,"data_stored":2747858944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2591358976,"data_stored":2591358976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3911122944,"data_stored":3911122944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2667511808,"data_stored":2667511808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4258267136,"data_stored":4258267136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4746706944,"data_stored":4746706944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:26:30.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:30.595 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:30.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:30.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:30.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 109386 109370 84.3709 65.1875 0.0100276 0.0118472 2024-09-14T04:26:30.814 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:30.840 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[112165]: osdmap e426: 8 total, 8 up, 6 in 2024-09-14T04:26:30.840 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:30.840 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:30.840 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[107399]: osdmap e426: 8 total, 8 up, 6 in 2024-09-14T04:26:30.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:30.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:30 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:30.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:30.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:30.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:31.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:30 smithi012 ceph-mon[101180]: osdmap e426: 8 total, 8 up, 6 in 2024-09-14T04:26:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:30 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:30 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:31.114 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:31.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:31.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:31.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:31.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:31.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:31.522 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:31.559 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:31.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:31.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:31.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:31.800 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 110408 110392 84.1209 63.875 0.0110936 0.0118827 2024-09-14T04:26:31.843 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:31.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[112165]: pgmap v1415: 109 pgs: 109 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:31 smithi001 ceph-mon[107399]: pgmap v1415: 109 pgs: 109 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:31.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:31.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:31.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:32.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:26:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:31 smithi012 ceph-mon[101180]: pgmap v1415: 109 pgs: 109 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:32.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:32.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:32.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:32.278 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:32.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:32.408 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:32.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:32.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:32.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:32.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:32.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:32.787 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:32.802 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 111509 111493 83.9342 68.8125 0.018517 0.0119085 2024-09-14T04:26:32.828 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:32.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:32.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:32.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:33.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:33.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:33.195 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:33.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:33.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:33.359 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:33.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:33.483 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:33.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:26:33.585 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:33.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:33.797 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:33.802 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 112144 112128 83.4072 39.6875 0.0140876 0.011984 2024-09-14T04:26:33.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:33.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:33.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:34.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:34.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:34.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:34.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:34.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:34.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:34.574 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:34.604 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:34.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:34.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:34.803 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 113285 113269 83.2644 71.3125 0.0206855 0.0120048 2024-09-14T04:26:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:34 smithi012 ceph-mon[101180]: pgmap v1416: 109 pgs: 109 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:34.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:34.886 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:34.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:34.888 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:34 smithi001 ceph-mon[107399]: pgmap v1416: 109 pgs: 109 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:34.888 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:34 smithi001 ceph-mon[112165]: pgmap v1416: 109 pgs: 109 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:26:34.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:35.019 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:35.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:35.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:35.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:35.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:35.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:35.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:35.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:35.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:35.668 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:35.710 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:35.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:35.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:35.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:35.803 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 114300 114284 83.0338 63.4375 0.0140078 0.0120378 2024-09-14T04:26:35.811 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:35.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:35.870 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:35.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:35.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:36.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:36.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:36.106 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:36.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:36.245 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:36.284 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:36.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:36.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:36.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:36.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:36.449 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:36.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.584 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:36.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:36.607 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:26:36.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:36.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:36.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:36.803 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 115372 115356 82.8495 67 0.0199229 0.0120648 2024-09-14T04:26:36.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:36 smithi012 ceph-mon[101180]: pgmap v1417: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3265937798' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:36 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:36.865 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:36] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.51.0" 2024-09-14T04:26:36.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[112165]: pgmap v1417: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:36.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3265937798' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.865 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[107399]: pgmap v1417: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:36.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3265937798' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:36 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:26:36.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:36.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:36.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:37.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:37.127 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:37.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:37.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:37.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:37.390 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:37.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:37.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:37.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:37.661 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:37.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[107399]: osdmap e427: 8 total, 8 up, 6 in 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:26:37.792 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:37 smithi001 ceph-mon[112165]: osdmap e427: 8 total, 8 up, 6 in 2024-09-14T04:26:37.804 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 116378 116362 82.6224 62.875 0.00700215 0.0120977 2024-09-14T04:26:37.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:37.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:37 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:37 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:26:37.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:37 smithi012 ceph-mon[101180]: osdmap e427: 8 total, 8 up, 6 in 2024-09-14T04:26:37.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:37.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:37.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:38.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:38.063 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:26:38.063 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:26:38.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:38.114 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:26:38.115 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:26:38.162 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:26:38.163 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:26:38.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:38.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:38.208 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:26:38.208 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:26:38.245 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:26:38.246 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:26:38.284 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:26:38.285 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:26:38.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:38.317 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:26:38.317 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:26:38.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:38.352 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:26:38.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:26:38.396 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:26:38.396 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:26:38.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:38.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:38.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:38.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:38.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:38.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:38.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:38.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:38 smithi001 ceph-mon[107399]: pgmap v1419: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:38.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/503524146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:38.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:38 smithi001 ceph-mon[112165]: pgmap v1419: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:38.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/503524146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:38.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:38.803 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 117242 117226 82.3008 54 0.0203772 0.0121454 2024-09-14T04:26:38.870 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:38.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:38.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:39.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:39.028 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:38 smithi012 ceph-mon[101180]: pgmap v1419: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:26:39.028 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/503524146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:39.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:39.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:39.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:39.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:39.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:39.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:26:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.448 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:39.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:39.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:39.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:39.735 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:39.804 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0087 2024-09-14T04:26:39.804 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 118241 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 82.1039 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 39.7209 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 223.688 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1313 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 635.54 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3579 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.0121754 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.0405113 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 2.49143 2024-09-14T04:26:39.805 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00114104 2024-09-14T04:26:39.809 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:26:39.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:39.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:39.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:39.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:40.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:40.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:40.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:40.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:40.297 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:40.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:40.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:40.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:40.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.581 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:40.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:40.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:40.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:40 smithi001 ceph-mon[112165]: pgmap v1420: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:26:40.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:40 smithi001 ceph-mon[107399]: pgmap v1420: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:26:40.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.722 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:40.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:40.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:40.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:40.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:40.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:40.965 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:40 smithi012 ceph-mon[101180]: pgmap v1420: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:26:41.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:41.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:41.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:41.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:41.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:41.275 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:41.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:41.429 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:41.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:41.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.565 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:41.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:41.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:41.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:41.771 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:26:41.777 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:26:41.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:41.820 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:41.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:41.949 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:41.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:42.051 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:42.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:42.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:42.238 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:42.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:42.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:42.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:42.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:26:42.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:42.511 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:42.613 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.662 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:42.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:42.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:42.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:42.791 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1421,"stamp":"2024-09-14T04:26:41.517271+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7655293136,"num_objects":116828,"num_object_clones":0,"num_object_copies":350484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116828,"num_whiteouts":0,"num_read":18,"num_read_kb":39,"num_write":118208,"num_write_kb":7518032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5441,"num_bytes_recovered":354709755,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95722,"ondisk_log_size":95722,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":27462904,"kb_used_data":22724992,"kb_used_omap":38,"kb_used_meta":4737625,"kb_avail":535008008,"statfs":{"total":575970213888,"available":547848200192,"internally_reserved":0,"allocated":23270391808,"data_stored":23253793195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39753,"internal_metadata":4851328183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":557449240,"num_objects":8507,"num_object_clones":0,"num_object_copies":25521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8507,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9839,"num_write_kb":587009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006210"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":657,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.606756+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:26:36.606756+0000","last_peered":"2024-09-14T04:26:36.606756+0000","last_clean":"2024-09-14T04:26:36.606756+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:26:36.606756+0000","last_undegraded":"2024-09-14T04:26:36.606756+0000","last_fullsized":"2024-09-14T04:26:36.606756+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":163,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618733+0000","last_change":"2024-09-14T04:23:28.577777+0000","last_active":"2024-09-14T04:26:36.618733+0000","last_peered":"2024-09-14T04:26:36.618733+0000","last_clean":"2024-09-14T04:26:36.618733+0000","last_became_active":"2024-09-14T04:23:28.577155+0000","last_became_peered":"2024-09-14T04:23:28.577155+0000","last_unstale":"2024-09-14T04:26:36.618733+0000","last_undegraded":"2024-09-14T04:26:36.618733+0000","last_fullsized":"2024-09-14T04:26:36.618733+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:01.212541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":175,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647445+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:26:29.647445+0000","last_peered":"2024-09-14T04:26:29.647445+0000","last_clean":"2024-09-14T04:26:29.647445+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:26:29.647445+0000","last_undegraded":"2024-09-14T04:26:29.647445+0000","last_fullsized":"2024-09-14T04:26:29.647445+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":765,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643865+0000","last_change":"2024-09-14T04:24:49.569053+0000","last_active":"2024-09-14T04:26:29.643865+0000","last_peered":"2024-09-14T04:26:29.643865+0000","last_clean":"2024-09-14T04:26:29.643865+0000","last_became_active":"2024-09-14T04:24:49.568690+0000","last_became_peered":"2024-09-14T04:24:49.568690+0000","last_unstale":"2024-09-14T04:26:29.643865+0000","last_undegraded":"2024-09-14T04:26:29.643865+0000","last_fullsized":"2024-09-14T04:26:29.643865+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:19:37.952706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644601+0000","last_change":"2024-09-14T04:25:25.582150+0000","last_active":"2024-09-14T04:26:29.644601+0000","last_peered":"2024-09-14T04:26:29.644601+0000","last_clean":"2024-09-14T04:26:29.644601+0000","last_became_active":"2024-09-14T04:25:25.581683+0000","last_became_peered":"2024-09-14T04:25:25.581683+0000","last_unstale":"2024-09-14T04:26:29.644601+0000","last_undegraded":"2024-09-14T04:26:29.644601+0000","last_fullsized":"2024-09-14T04:26:29.644601+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:10.188214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":473,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.603848+0000","last_change":"2024-09-14T04:25:35.562683+0000","last_active":"2024-09-14T04:26:36.603848+0000","last_peered":"2024-09-14T04:26:36.603848+0000","last_clean":"2024-09-14T04:26:36.603848+0000","last_became_active":"2024-09-14T04:25:35.562281+0000","last_became_peered":"2024-09-14T04:25:35.562281+0000","last_unstale":"2024-09-14T04:26:36.603848+0000","last_undegraded":"2024-09-14T04:26:36.603848+0000","last_fullsized":"2024-09-14T04:26:36.603848+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:09.493757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":947,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614133+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:26:36.614133+0000","last_peered":"2024-09-14T04:26:36.614133+0000","last_clean":"2024-09-14T04:26:36.614133+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:26:36.614133+0000","last_undegraded":"2024-09-14T04:26:36.614133+0000","last_fullsized":"2024-09-14T04:26:36.614133+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":459,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612527+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:26:36.612527+0000","last_peered":"2024-09-14T04:26:36.612527+0000","last_clean":"2024-09-14T04:26:36.612527+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:26:36.612527+0000","last_undegraded":"2024-09-14T04:26:36.612527+0000","last_fullsized":"2024-09-14T04:26:36.612527+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":381,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647100+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:26:29.647100+0000","last_peered":"2024-09-14T04:26:29.647100+0000","last_clean":"2024-09-14T04:26:29.647100+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:26:29.647100+0000","last_undegraded":"2024-09-14T04:26:29.647100+0000","last_fullsized":"2024-09-14T04:26:29.647100+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":593,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643766+0000","last_change":"2024-09-14T04:24:49.572838+0000","last_active":"2024-09-14T04:26:29.643766+0000","last_peered":"2024-09-14T04:26:29.643766+0000","last_clean":"2024-09-14T04:26:29.643766+0000","last_became_active":"2024-09-14T04:24:49.572388+0000","last_became_peered":"2024-09-14T04:24:49.572388+0000","last_unstale":"2024-09-14T04:26:29.643766+0000","last_undegraded":"2024-09-14T04:26:29.643766+0000","last_fullsized":"2024-09-14T04:26:29.643766+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:22.578056+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617890+0000","last_change":"2024-09-14T04:25:35.565538+0000","last_active":"2024-09-14T04:26:36.617890+0000","last_peered":"2024-09-14T04:26:36.617890+0000","last_clean":"2024-09-14T04:26:36.617890+0000","last_became_active":"2024-09-14T04:25:35.565210+0000","last_became_peered":"2024-09-14T04:25:35.565210+0000","last_unstale":"2024-09-14T04:26:36.617890+0000","last_undegraded":"2024-09-14T04:26:36.617890+0000","last_fullsized":"2024-09-14T04:26:36.617890+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:08.538693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":541,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610085+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:26:36.610085+0000","last_peered":"2024-09-14T04:26:36.610085+0000","last_clean":"2024-09-14T04:26:36.610085+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:26:36.610085+0000","last_undegraded":"2024-09-14T04:26:36.610085+0000","last_fullsized":"2024-09-14T04:26:36.610085+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"427'7029","reported_seq":7098,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:37.239874+0000","last_change":"2024-09-14T04:25:02.146250+0000","last_active":"2024-09-14T04:26:37.239874+0000","last_peered":"2024-09-14T04:26:37.239874+0000","last_clean":"2024-09-14T04:26:37.239874+0000","last_became_active":"2024-09-14T04:25:02.145730+0000","last_became_peered":"2024-09-14T04:25:02.145730+0000","last_unstale":"2024-09-14T04:26:37.239874+0000","last_undegraded":"2024-09-14T04:26:37.239874+0000","last_fullsized":"2024-09-14T04:26:37.239874+0000","mapping_epoch":398,"log_start":"406'500","ondisk_log_start":"406'500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:04.224581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":460652544,"num_objects":7029,"num_object_clones":0,"num_object_copies":21087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7029,"num_write_kb":449856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":749,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647690+0000","last_change":"2024-09-14T04:24:49.572850+0000","last_active":"2024-09-14T04:26:29.647690+0000","last_peered":"2024-09-14T04:26:29.647690+0000","last_clean":"2024-09-14T04:26:29.647690+0000","last_became_active":"2024-09-14T04:24:49.572405+0000","last_became_peered":"2024-09-14T04:24:49.572405+0000","last_unstale":"2024-09-14T04:26:29.647690+0000","last_undegraded":"2024-09-14T04:26:29.647690+0000","last_fullsized":"2024-09-14T04:26:29.647690+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:06.709118+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":143,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647932+0000","last_change":"2024-09-14T04:23:28.575634+0000","last_active":"2024-09-14T04:26:29.647932+0000","last_peered":"2024-09-14T04:26:29.647932+0000","last_clean":"2024-09-14T04:26:29.647932+0000","last_became_active":"2024-09-14T04:23:28.558391+0000","last_became_peered":"2024-09-14T04:23:28.558391+0000","last_unstale":"2024-09-14T04:26:29.647932+0000","last_undegraded":"2024-09-14T04:26:29.647932+0000","last_fullsized":"2024-09-14T04:26:29.647932+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:03.845925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643649+0000","last_change":"2024-09-14T04:25:52.595074+0000","last_active":"2024-09-14T04:26:29.643649+0000","last_peered":"2024-09-14T04:26:29.643649+0000","last_clean":"2024-09-14T04:26:29.643649+0000","last_became_active":"2024-09-14T04:25:52.594744+0000","last_became_peered":"2024-09-14T04:25:52.594744+0000","last_unstale":"2024-09-14T04:26:29.643649+0000","last_undegraded":"2024-09-14T04:26:29.643649+0000","last_fullsized":"2024-09-14T04:26:29.643649+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:10.267321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"427'7498","reported_seq":7567,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.107447+0000","last_change":"2024-09-14T04:25:01.992464+0000","last_active":"2024-09-14T04:26:41.107447+0000","last_peered":"2024-09-14T04:26:41.107447+0000","last_clean":"2024-09-14T04:26:41.107447+0000","last_became_active":"2024-09-14T04:25:01.992117+0000","last_became_peered":"2024-09-14T04:25:01.992117+0000","last_unstale":"2024-09-14T04:26:41.107447+0000","last_undegraded":"2024-09-14T04:26:41.107447+0000","last_fullsized":"2024-09-14T04:26:41.107447+0000","mapping_epoch":398,"log_start":"409'2300","ondisk_log_start":"409'2300","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5198,"log_dups_size":0,"ondisk_log_size":5198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:21.289671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479199232,"num_objects":7312,"num_object_clones":0,"num_object_copies":21936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7498,"num_write_kb":473920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1000,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617877+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:26:36.617877+0000","last_peered":"2024-09-14T04:26:36.617877+0000","last_clean":"2024-09-14T04:26:36.617877+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:26:36.617877+0000","last_undegraded":"2024-09-14T04:26:36.617877+0000","last_fullsized":"2024-09-14T04:26:36.617877+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":40,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646979+0000","last_change":"2024-09-14T04:25:07.963635+0000","last_active":"2024-09-14T04:26:29.646979+0000","last_peered":"2024-09-14T04:26:29.646979+0000","last_clean":"2024-09-14T04:26:29.646979+0000","last_became_active":"2024-09-14T04:25:07.959540+0000","last_became_peered":"2024-09-14T04:25:07.959540+0000","last_unstale":"2024-09-14T04:26:29.646979+0000","last_undegraded":"2024-09-14T04:26:29.646979+0000","last_fullsized":"2024-09-14T04:26:29.646979+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:05.353685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":453,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646861+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:26:29.646861+0000","last_peered":"2024-09-14T04:26:29.646861+0000","last_clean":"2024-09-14T04:26:29.646861+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:26:29.646861+0000","last_undegraded":"2024-09-14T04:26:29.646861+0000","last_fullsized":"2024-09-14T04:26:29.646861+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"427'7423","reported_seq":7492,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:39.489049+0000","last_change":"2024-09-14T04:25:02.146419+0000","last_active":"2024-09-14T04:26:39.489049+0000","last_peered":"2024-09-14T04:26:39.489049+0000","last_clean":"2024-09-14T04:26:39.489049+0000","last_became_active":"2024-09-14T04:25:02.145877+0000","last_became_peered":"2024-09-14T04:25:02.145877+0000","last_unstale":"2024-09-14T04:26:39.489049+0000","last_undegraded":"2024-09-14T04:26:39.489049+0000","last_fullsized":"2024-09-14T04:26:39.489049+0000","mapping_epoch":398,"log_start":"406'1900","ondisk_log_start":"406'1900","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:31.774348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486473728,"num_objects":7423,"num_object_clones":0,"num_object_copies":22269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7423,"num_write_kb":475072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":449,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644950+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:26:29.644950+0000","last_peered":"2024-09-14T04:26:29.644950+0000","last_clean":"2024-09-14T04:26:29.644950+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:26:29.644950+0000","last_undegraded":"2024-09-14T04:26:29.644950+0000","last_fullsized":"2024-09-14T04:26:29.644950+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":34,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644485+0000","last_change":"2024-09-14T04:25:25.584570+0000","last_active":"2024-09-14T04:26:29.644485+0000","last_peered":"2024-09-14T04:26:29.644485+0000","last_clean":"2024-09-14T04:26:29.644485+0000","last_became_active":"2024-09-14T04:25:25.581721+0000","last_became_peered":"2024-09-14T04:25:25.581721+0000","last_unstale":"2024-09-14T04:26:29.644485+0000","last_undegraded":"2024-09-14T04:26:29.644485+0000","last_fullsized":"2024-09-14T04:26:29.644485+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:59.618866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":446,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617461+0000","last_change":"2024-09-14T04:24:08.985773+0000","last_active":"2024-09-14T04:26:36.617461+0000","last_peered":"2024-09-14T04:26:36.617461+0000","last_clean":"2024-09-14T04:26:36.617461+0000","last_became_active":"2024-09-14T04:24:08.985458+0000","last_became_peered":"2024-09-14T04:24:08.985458+0000","last_unstale":"2024-09-14T04:26:36.617461+0000","last_undegraded":"2024-09-14T04:26:36.617461+0000","last_fullsized":"2024-09-14T04:26:36.617461+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:10.470781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"427'7514","reported_seq":13771,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.081987+0000","last_change":"2024-09-14T04:26:29.659059+0000","last_active":"2024-09-14T04:26:41.081987+0000","last_peered":"2024-09-14T04:26:41.081987+0000","last_clean":"2024-09-14T04:26:41.081987+0000","last_became_active":"2024-09-14T04:26:29.648121+0000","last_became_peered":"2024-09-14T04:26:29.648121+0000","last_unstale":"2024-09-14T04:26:41.081987+0000","last_undegraded":"2024-09-14T04:26:41.081987+0000","last_fullsized":"2024-09-14T04:26:41.081987+0000","mapping_epoch":425,"log_start":"406'1700","ondisk_log_start":"406'1700","created":398,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:05.353280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":478281728,"num_objects":7298,"num_object_clones":0,"num_object_copies":21894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7514,"num_write_kb":473984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":403,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647501+0000","last_change":"2024-09-14T04:25:10.210264+0000","last_active":"2024-09-14T04:26:29.647501+0000","last_peered":"2024-09-14T04:26:29.647501+0000","last_clean":"2024-09-14T04:26:29.647501+0000","last_became_active":"2024-09-14T04:25:10.209992+0000","last_became_peered":"2024-09-14T04:25:10.209992+0000","last_unstale":"2024-09-14T04:26:29.647501+0000","last_undegraded":"2024-09-14T04:26:29.647501+0000","last_fullsized":"2024-09-14T04:26:29.647501+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:50.222369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:50.222369+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":376,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644436+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:26:29.644436+0000","last_peered":"2024-09-14T04:26:29.644436+0000","last_clean":"2024-09-14T04:26:29.644436+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:26:29.644436+0000","last_undegraded":"2024-09-14T04:26:29.644436+0000","last_fullsized":"2024-09-14T04:26:29.644436+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":453,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610959+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:26:36.610959+0000","last_peered":"2024-09-14T04:26:36.610959+0000","last_clean":"2024-09-14T04:26:36.610959+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:26:36.610959+0000","last_undegraded":"2024-09-14T04:26:36.610959+0000","last_fullsized":"2024-09-14T04:26:36.610959+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"427'7482","reported_seq":7520,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:40.085937+0000","last_change":"2024-09-14T04:25:01.992668+0000","last_active":"2024-09-14T04:26:40.085937+0000","last_peered":"2024-09-14T04:26:40.085937+0000","last_clean":"2024-09-14T04:26:40.085937+0000","last_became_active":"2024-09-14T04:25:01.992359+0000","last_became_peered":"2024-09-14T04:25:01.992359+0000","last_unstale":"2024-09-14T04:26:40.085937+0000","last_undegraded":"2024-09-14T04:26:40.085937+0000","last_fullsized":"2024-09-14T04:26:40.085937+0000","mapping_epoch":398,"log_start":"409'2300","ondisk_log_start":"409'2300","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:08.434277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486670336,"num_objects":7426,"num_object_clones":0,"num_object_copies":22278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7482,"num_write_kb":477056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":741,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608297+0000","last_change":"2024-09-14T04:24:10.193742+0000","last_active":"2024-09-14T04:26:36.608297+0000","last_peered":"2024-09-14T04:26:36.608297+0000","last_clean":"2024-09-14T04:26:36.608297+0000","last_became_active":"2024-09-14T04:24:10.193277+0000","last_became_peered":"2024-09-14T04:24:10.193277+0000","last_unstale":"2024-09-14T04:26:36.608297+0000","last_undegraded":"2024-09-14T04:26:36.608297+0000","last_fullsized":"2024-09-14T04:26:36.608297+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:08.270894+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":457,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610761+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:26:36.610761+0000","last_peered":"2024-09-14T04:26:36.610761+0000","last_clean":"2024-09-14T04:26:36.610761+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:26:36.610761+0000","last_undegraded":"2024-09-14T04:26:36.610761+0000","last_fullsized":"2024-09-14T04:26:36.610761+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":428,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647360+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:26:29.647360+0000","last_peered":"2024-09-14T04:26:29.647360+0000","last_clean":"2024-09-14T04:26:29.647360+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:26:29.647360+0000","last_undegraded":"2024-09-14T04:26:29.647360+0000","last_fullsized":"2024-09-14T04:26:29.647360+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"427'7327","reported_seq":7396,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:37.227937+0000","last_change":"2024-09-14T04:25:01.995435+0000","last_active":"2024-09-14T04:26:37.227937+0000","last_peered":"2024-09-14T04:26:37.227937+0000","last_clean":"2024-09-14T04:26:37.227937+0000","last_became_active":"2024-09-14T04:25:01.995177+0000","last_became_peered":"2024-09-14T04:25:01.995177+0000","last_unstale":"2024-09-14T04:26:37.227937+0000","last_undegraded":"2024-09-14T04:26:37.227937+0000","last_fullsized":"2024-09-14T04:26:37.227937+0000","mapping_epoch":398,"log_start":"406'800","ondisk_log_start":"406'800","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:02.315254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480182272,"num_objects":7327,"num_object_clones":0,"num_object_copies":21981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7327,"num_write_kb":468928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":661,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643688+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:26:29.643688+0000","last_peered":"2024-09-14T04:26:29.643688+0000","last_clean":"2024-09-14T04:26:29.643688+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:26:29.643688+0000","last_undegraded":"2024-09-14T04:26:29.643688+0000","last_fullsized":"2024-09-14T04:26:29.643688+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":535,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.604157+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:26:36.604157+0000","last_peered":"2024-09-14T04:26:36.604157+0000","last_clean":"2024-09-14T04:26:36.604157+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:26:36.604157+0000","last_undegraded":"2024-09-14T04:26:36.604157+0000","last_fullsized":"2024-09-14T04:26:36.604157+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647867+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:26:29.647867+0000","last_peered":"2024-09-14T04:26:29.647867+0000","last_clean":"2024-09-14T04:26:29.647867+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:26:29.647867+0000","last_undegraded":"2024-09-14T04:26:29.647867+0000","last_fullsized":"2024-09-14T04:26:29.647867+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"427'7469","reported_seq":12084,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:40.092201+0000","last_change":"2024-09-14T04:25:44.643768+0000","last_active":"2024-09-14T04:26:40.092201+0000","last_peered":"2024-09-14T04:26:40.092201+0000","last_clean":"2024-09-14T04:26:40.092201+0000","last_became_active":"2024-09-14T04:25:44.638986+0000","last_became_peered":"2024-09-14T04:25:44.638986+0000","last_unstale":"2024-09-14T04:26:40.092201+0000","last_undegraded":"2024-09-14T04:26:40.092201+0000","last_fullsized":"2024-09-14T04:26:40.092201+0000","mapping_epoch":419,"log_start":"409'2300","ondisk_log_start":"409'2300","created":398,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5169,"log_dups_size":0,"ondisk_log_size":5169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:07.177695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486342656,"num_objects":7421,"num_object_clones":0,"num_object_copies":22263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7469,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":927,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613842+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:26:36.613842+0000","last_peered":"2024-09-14T04:26:36.613842+0000","last_clean":"2024-09-14T04:26:36.613842+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:26:36.613842+0000","last_undegraded":"2024-09-14T04:26:36.613842+0000","last_fullsized":"2024-09-14T04:26:36.613842+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647166+0000","last_change":"2024-09-14T04:25:07.967648+0000","last_active":"2024-09-14T04:26:29.647166+0000","last_peered":"2024-09-14T04:26:29.647166+0000","last_clean":"2024-09-14T04:26:29.647166+0000","last_became_active":"2024-09-14T04:25:07.967284+0000","last_became_peered":"2024-09-14T04:25:07.967284+0000","last_unstale":"2024-09-14T04:26:29.647166+0000","last_undegraded":"2024-09-14T04:26:29.647166+0000","last_fullsized":"2024-09-14T04:26:29.647166+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:29.967190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":434,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647637+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:26:29.647637+0000","last_peered":"2024-09-14T04:26:29.647637+0000","last_clean":"2024-09-14T04:26:29.647637+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:26:29.647637+0000","last_undegraded":"2024-09-14T04:26:29.647637+0000","last_fullsized":"2024-09-14T04:26:29.647637+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"427'7420","reported_seq":7458,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:40.050448+0000","last_change":"2024-09-14T04:25:02.146105+0000","last_active":"2024-09-14T04:26:40.050448+0000","last_peered":"2024-09-14T04:26:40.050448+0000","last_clean":"2024-09-14T04:26:40.050448+0000","last_became_active":"2024-09-14T04:25:02.145736+0000","last_became_peered":"2024-09-14T04:25:02.145736+0000","last_unstale":"2024-09-14T04:26:40.050448+0000","last_undegraded":"2024-09-14T04:26:40.050448+0000","last_fullsized":"2024-09-14T04:26:40.050448+0000","mapping_epoch":398,"log_start":"409'2300","ondisk_log_start":"409'2300","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.907164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482869248,"num_objects":7368,"num_object_clones":0,"num_object_copies":22104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7420,"num_write_kb":473216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":632,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644039+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:26:29.644039+0000","last_peered":"2024-09-14T04:26:29.644039+0000","last_clean":"2024-09-14T04:26:29.644039+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:26:29.644039+0000","last_undegraded":"2024-09-14T04:26:29.644039+0000","last_fullsized":"2024-09-14T04:26:29.644039+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609291+0000","last_change":"2024-09-14T04:25:35.561629+0000","last_active":"2024-09-14T04:26:36.609291+0000","last_peered":"2024-09-14T04:26:36.609291+0000","last_clean":"2024-09-14T04:26:36.609291+0000","last_became_active":"2024-09-14T04:25:35.561305+0000","last_became_peered":"2024-09-14T04:25:35.561305+0000","last_unstale":"2024-09-14T04:26:36.609291+0000","last_undegraded":"2024-09-14T04:26:36.609291+0000","last_fullsized":"2024-09-14T04:26:36.609291+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:39.574054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":696,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614968+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:26:36.614968+0000","last_peered":"2024-09-14T04:26:36.614968+0000","last_clean":"2024-09-14T04:26:36.614968+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:26:36.614968+0000","last_undegraded":"2024-09-14T04:26:36.614968+0000","last_fullsized":"2024-09-14T04:26:36.614968+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"427'7618","reported_seq":7687,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.126641+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:26:41.126641+0000","last_peered":"2024-09-14T04:26:41.126641+0000","last_clean":"2024-09-14T04:26:41.126641+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:26:41.126641+0000","last_undegraded":"2024-09-14T04:26:41.126641+0000","last_fullsized":"2024-09-14T04:26:41.126641+0000","mapping_epoch":398,"log_start":"410'2400","ondisk_log_start":"410'2400","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5218,"log_dups_size":0,"ondisk_log_size":5218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:09.181654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486932480,"num_objects":7430,"num_object_clones":0,"num_object_copies":22290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7618,"num_write_kb":481536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":796,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610250+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:26:36.610250+0000","last_peered":"2024-09-14T04:26:36.610250+0000","last_clean":"2024-09-14T04:26:36.610250+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:26:36.610250+0000","last_undegraded":"2024-09-14T04:26:36.610250+0000","last_fullsized":"2024-09-14T04:26:36.610250+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:17.530093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":366,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644179+0000","last_change":"2024-09-14T04:24:08.979706+0000","last_active":"2024-09-14T04:26:29.644179+0000","last_peered":"2024-09-14T04:26:29.644179+0000","last_clean":"2024-09-14T04:26:29.644179+0000","last_became_active":"2024-09-14T04:24:08.979397+0000","last_became_peered":"2024-09-14T04:24:08.979397+0000","last_unstale":"2024-09-14T04:26:29.644179+0000","last_undegraded":"2024-09-14T04:26:29.644179+0000","last_fullsized":"2024-09-14T04:26:29.644179+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:12.232467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609238+0000","last_change":"2024-09-14T04:25:52.594667+0000","last_active":"2024-09-14T04:26:36.609238+0000","last_peered":"2024-09-14T04:26:36.609238+0000","last_clean":"2024-09-14T04:26:36.609238+0000","last_became_active":"2024-09-14T04:25:52.594400+0000","last_became_peered":"2024-09-14T04:25:52.594400+0000","last_unstale":"2024-09-14T04:26:36.609238+0000","last_undegraded":"2024-09-14T04:26:36.609238+0000","last_fullsized":"2024-09-14T04:26:36.609238+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:12.710417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647785+0000","last_change":"2024-09-14T04:25:35.560355+0000","last_active":"2024-09-14T04:26:29.647785+0000","last_peered":"2024-09-14T04:26:29.647785+0000","last_clean":"2024-09-14T04:26:29.647785+0000","last_became_active":"2024-09-14T04:25:35.559975+0000","last_became_peered":"2024-09-14T04:25:35.559975+0000","last_unstale":"2024-09-14T04:26:29.647785+0000","last_undegraded":"2024-09-14T04:26:29.647785+0000","last_fullsized":"2024-09-14T04:26:29.647785+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:58.033714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"427'7215","reported_seq":7289,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:37.280427+0000","last_change":"2024-09-14T04:25:07.966414+0000","last_active":"2024-09-14T04:26:37.280427+0000","last_peered":"2024-09-14T04:26:37.280427+0000","last_clean":"2024-09-14T04:26:37.280427+0000","last_became_active":"2024-09-14T04:25:07.966014+0000","last_became_peered":"2024-09-14T04:25:07.966014+0000","last_unstale":"2024-09-14T04:26:37.280427+0000","last_undegraded":"2024-09-14T04:26:37.280427+0000","last_fullsized":"2024-09-14T04:26:37.280427+0000","mapping_epoch":401,"log_start":"406'600","ondisk_log_start":"406'600","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:58.062778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":472842240,"num_objects":7215,"num_object_clones":0,"num_object_copies":21645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7215,"num_write_kb":461760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":918,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613940+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:26:36.613940+0000","last_peered":"2024-09-14T04:26:36.613940+0000","last_clean":"2024-09-14T04:26:36.613940+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:26:36.613940+0000","last_undegraded":"2024-09-14T04:26:36.613940+0000","last_fullsized":"2024-09-14T04:26:36.613940+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":397,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613255+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:26:36.613255+0000","last_peered":"2024-09-14T04:26:36.613255+0000","last_clean":"2024-09-14T04:26:36.613255+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:26:36.613255+0000","last_undegraded":"2024-09-14T04:26:36.613255+0000","last_fullsized":"2024-09-14T04:26:36.613255+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":702,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.606712+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:26:36.606712+0000","last_peered":"2024-09-14T04:26:36.606712+0000","last_clean":"2024-09-14T04:26:36.606712+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:26:36.606712+0000","last_undegraded":"2024-09-14T04:26:36.606712+0000","last_fullsized":"2024-09-14T04:26:36.606712+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":464,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612605+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:26:36.612605+0000","last_peered":"2024-09-14T04:26:36.612605+0000","last_clean":"2024-09-14T04:26:36.612605+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:26:36.612605+0000","last_undegraded":"2024-09-14T04:26:36.612605+0000","last_fullsized":"2024-09-14T04:26:36.612605+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"427'7395","reported_seq":7433,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.072492+0000","last_change":"2024-09-14T04:25:01.992854+0000","last_active":"2024-09-14T04:26:41.072492+0000","last_peered":"2024-09-14T04:26:41.072492+0000","last_clean":"2024-09-14T04:26:41.072492+0000","last_became_active":"2024-09-14T04:25:01.992514+0000","last_became_peered":"2024-09-14T04:25:01.992514+0000","last_unstale":"2024-09-14T04:26:41.072492+0000","last_undegraded":"2024-09-14T04:26:41.072492+0000","last_fullsized":"2024-09-14T04:26:41.072492+0000","mapping_epoch":398,"log_start":"406'1600","ondisk_log_start":"406'1600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5795,"log_dups_size":0,"ondisk_log_size":5795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:59.789126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":471793664,"num_objects":7199,"num_object_clones":0,"num_object_copies":21597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7395,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":671,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.604014+0000","last_change":"2024-09-14T04:25:10.213700+0000","last_active":"2024-09-14T04:26:36.604014+0000","last_peered":"2024-09-14T04:26:36.604014+0000","last_clean":"2024-09-14T04:26:36.604014+0000","last_became_active":"2024-09-14T04:25:10.213291+0000","last_became_peered":"2024-09-14T04:25:10.213291+0000","last_unstale":"2024-09-14T04:26:36.604014+0000","last_undegraded":"2024-09-14T04:26:36.604014+0000","last_fullsized":"2024-09-14T04:26:36.604014+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:58.394641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":670,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608438+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:26:36.608438+0000","last_peered":"2024-09-14T04:26:36.608438+0000","last_clean":"2024-09-14T04:26:36.608438+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:26:36.608438+0000","last_undegraded":"2024-09-14T04:26:36.608438+0000","last_fullsized":"2024-09-14T04:26:36.608438+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":701,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613398+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:26:36.613398+0000","last_peered":"2024-09-14T04:26:36.613398+0000","last_clean":"2024-09-14T04:26:36.613398+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:26:36.613398+0000","last_undegraded":"2024-09-14T04:26:36.613398+0000","last_fullsized":"2024-09-14T04:26:36.613398+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614777+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:26:36.614777+0000","last_peered":"2024-09-14T04:26:36.614777+0000","last_clean":"2024-09-14T04:26:36.614777+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:26:36.614777+0000","last_undegraded":"2024-09-14T04:26:36.614777+0000","last_fullsized":"2024-09-14T04:26:36.614777+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"427'7495","reported_seq":7534,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:40.108184+0000","last_change":"2024-09-14T04:25:02.146113+0000","last_active":"2024-09-14T04:26:40.108184+0000","last_peered":"2024-09-14T04:26:40.108184+0000","last_clean":"2024-09-14T04:26:40.108184+0000","last_became_active":"2024-09-14T04:25:02.145762+0000","last_became_peered":"2024-09-14T04:25:02.145762+0000","last_unstale":"2024-09-14T04:26:40.108184+0000","last_undegraded":"2024-09-14T04:26:40.108184+0000","last_fullsized":"2024-09-14T04:26:40.108184+0000","mapping_epoch":398,"log_start":"410'2400","ondisk_log_start":"410'2400","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5095,"log_dups_size":0,"ondisk_log_size":5095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:33.000554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":487981080,"num_objects":7447,"num_object_clones":0,"num_object_copies":22341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7495,"num_write_kb":478081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":655,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643642+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:26:29.643642+0000","last_peered":"2024-09-14T04:26:29.643642+0000","last_clean":"2024-09-14T04:26:29.643642+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:26:29.643642+0000","last_undegraded":"2024-09-14T04:26:29.643642+0000","last_fullsized":"2024-09-14T04:26:29.643642+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":363,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643889+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:26:29.643889+0000","last_peered":"2024-09-14T04:26:29.643889+0000","last_clean":"2024-09-14T04:26:29.643889+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:26:29.643889+0000","last_undegraded":"2024-09-14T04:26:29.643889+0000","last_fullsized":"2024-09-14T04:26:29.643889+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":446,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647506+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:26:29.647506+0000","last_peered":"2024-09-14T04:26:29.647506+0000","last_clean":"2024-09-14T04:26:29.647506+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:26:29.647506+0000","last_undegraded":"2024-09-14T04:26:29.647506+0000","last_fullsized":"2024-09-14T04:26:29.647506+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":383,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644535+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:26:29.644535+0000","last_peered":"2024-09-14T04:26:29.644535+0000","last_clean":"2024-09-14T04:26:29.644535+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:26:29.644535+0000","last_undegraded":"2024-09-14T04:26:29.644535+0000","last_fullsized":"2024-09-14T04:26:29.644535+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"427'7573","reported_seq":7611,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.023691+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:26:41.023691+0000","last_peered":"2024-09-14T04:26:41.023691+0000","last_clean":"2024-09-14T04:26:41.023691+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:26:41.023691+0000","last_undegraded":"2024-09-14T04:26:41.023691+0000","last_fullsized":"2024-09-14T04:26:41.023691+0000","mapping_epoch":398,"log_start":"406'1800","ondisk_log_start":"406'1800","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":485163008,"num_objects":7403,"num_object_clones":0,"num_object_copies":22209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7573,"num_write_kb":479232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":690,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617990+0000","last_change":"2024-09-14T04:25:38.633170+0000","last_active":"2024-09-14T04:26:36.617990+0000","last_peered":"2024-09-14T04:26:36.617990+0000","last_clean":"2024-09-14T04:26:36.617990+0000","last_became_active":"2024-09-14T04:25:38.632816+0000","last_became_peered":"2024-09-14T04:25:38.632816+0000","last_unstale":"2024-09-14T04:26:36.617990+0000","last_undegraded":"2024-09-14T04:26:36.617990+0000","last_fullsized":"2024-09-14T04:26:36.617990+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:55:39.218376+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":392,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644959+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:26:29.644959+0000","last_peered":"2024-09-14T04:26:29.644959+0000","last_clean":"2024-09-14T04:26:29.644959+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:26:29.644959+0000","last_undegraded":"2024-09-14T04:26:29.644959+0000","last_fullsized":"2024-09-14T04:26:29.644959+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":758,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618982+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:26:36.618982+0000","last_peered":"2024-09-14T04:26:36.618982+0000","last_clean":"2024-09-14T04:26:36.618982+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:26:36.618982+0000","last_undegraded":"2024-09-14T04:26:36.618982+0000","last_fullsized":"2024-09-14T04:26:36.618982+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617324+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:26:36.617324+0000","last_peered":"2024-09-14T04:26:36.617324+0000","last_clean":"2024-09-14T04:26:36.617324+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:26:36.617324+0000","last_undegraded":"2024-09-14T04:26:36.617324+0000","last_fullsized":"2024-09-14T04:26:36.617324+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"427'7159","reported_seq":7228,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:37.223425+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:26:37.223425+0000","last_peered":"2024-09-14T04:26:37.223425+0000","last_clean":"2024-09-14T04:26:37.223425+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:26:37.223425+0000","last_undegraded":"2024-09-14T04:26:37.223425+0000","last_fullsized":"2024-09-14T04:26:37.223425+0000","mapping_epoch":398,"log_start":"406'600","ondisk_log_start":"406'600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:13.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":469172224,"num_objects":7159,"num_object_clones":0,"num_object_copies":21477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7159,"num_write_kb":458176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":708,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612174+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:26:36.612174+0000","last_peered":"2024-09-14T04:26:36.612174+0000","last_clean":"2024-09-14T04:26:36.612174+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:26:36.612174+0000","last_undegraded":"2024-09-14T04:26:36.612174+0000","last_fullsized":"2024-09-14T04:26:36.612174+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":704,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618962+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:26:36.618962+0000","last_peered":"2024-09-14T04:26:36.618962+0000","last_clean":"2024-09-14T04:26:36.618962+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:26:36.618962+0000","last_undegraded":"2024-09-14T04:26:36.618962+0000","last_fullsized":"2024-09-14T04:26:36.618962+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":436,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618630+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:26:36.618630+0000","last_peered":"2024-09-14T04:26:36.618630+0000","last_clean":"2024-09-14T04:26:36.618630+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:26:36.618630+0000","last_undegraded":"2024-09-14T04:26:36.618630+0000","last_fullsized":"2024-09-14T04:26:36.618630+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":469,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610874+0000","last_change":"2024-09-14T04:24:08.989110+0000","last_active":"2024-09-14T04:26:36.610874+0000","last_peered":"2024-09-14T04:26:36.610874+0000","last_clean":"2024-09-14T04:26:36.610874+0000","last_became_active":"2024-09-14T04:24:08.987440+0000","last_became_peered":"2024-09-14T04:24:08.987440+0000","last_unstale":"2024-09-14T04:26:36.610874+0000","last_undegraded":"2024-09-14T04:26:36.610874+0000","last_fullsized":"2024-09-14T04:26:36.610874+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:17:32.812292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"427'7407","reported_seq":7445,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:41.071885+0000","last_change":"2024-09-14T04:25:01.993057+0000","last_active":"2024-09-14T04:26:41.071885+0000","last_peered":"2024-09-14T04:26:41.071885+0000","last_clean":"2024-09-14T04:26:41.071885+0000","last_became_active":"2024-09-14T04:25:01.992668+0000","last_became_peered":"2024-09-14T04:25:01.992668+0000","last_unstale":"2024-09-14T04:26:41.071885+0000","last_undegraded":"2024-09-14T04:26:41.071885+0000","last_fullsized":"2024-09-14T04:26:41.071885+0000","mapping_epoch":398,"log_start":"406'1600","ondisk_log_start":"406'1600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5807,"log_dups_size":0,"ondisk_log_size":5807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:38.544577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474152960,"num_objects":7235,"num_object_clones":0,"num_object_copies":21705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7407,"num_write_kb":468544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":732,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647627+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:26:29.647627+0000","last_peered":"2024-09-14T04:26:29.647627+0000","last_clean":"2024-09-14T04:26:29.647627+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:26:29.647627+0000","last_undegraded":"2024-09-14T04:26:29.647627+0000","last_fullsized":"2024-09-14T04:26:29.647627+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":359,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.611710+0000","last_change":"2024-09-14T04:24:08.999455+0000","last_active":"2024-09-14T04:26:36.611710+0000","last_peered":"2024-09-14T04:26:36.611710+0000","last_clean":"2024-09-14T04:26:36.611710+0000","last_became_active":"2024-09-14T04:24:08.994093+0000","last_became_peered":"2024-09-14T04:24:08.994093+0000","last_unstale":"2024-09-14T04:26:36.611710+0000","last_undegraded":"2024-09-14T04:26:36.611710+0000","last_fullsized":"2024-09-14T04:26:36.611710+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:29:31.948482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":690,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614508+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:26:36.614508+0000","last_peered":"2024-09-14T04:26:36.614508+0000","last_clean":"2024-09-14T04:26:36.614508+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:26:36.614508+0000","last_undegraded":"2024-09-14T04:26:36.614508+0000","last_fullsized":"2024-09-14T04:26:36.614508+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":389,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609977+0000","last_change":"2024-09-14T04:25:35.562908+0000","last_active":"2024-09-14T04:26:36.609977+0000","last_peered":"2024-09-14T04:26:36.609977+0000","last_clean":"2024-09-14T04:26:36.609977+0000","last_became_active":"2024-09-14T04:25:35.562023+0000","last_became_peered":"2024-09-14T04:25:35.562023+0000","last_unstale":"2024-09-14T04:26:36.609977+0000","last_undegraded":"2024-09-14T04:26:36.609977+0000","last_fullsized":"2024-09-14T04:26:36.609977+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:55.656734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.8","version":"427'7110","reported_seq":7179,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:37.253575+0000","last_change":"2024-09-14T04:25:01.996105+0000","last_active":"2024-09-14T04:26:37.253575+0000","last_peered":"2024-09-14T04:26:37.253575+0000","last_clean":"2024-09-14T04:26:37.253575+0000","last_became_active":"2024-09-14T04:25:01.995811+0000","last_became_peered":"2024-09-14T04:25:01.995811+0000","last_unstale":"2024-09-14T04:26:37.253575+0000","last_undegraded":"2024-09-14T04:26:37.253575+0000","last_fullsized":"2024-09-14T04:26:37.253575+0000","mapping_epoch":398,"log_start":"406'500","ondisk_log_start":"406'500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:04.759721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":465960960,"num_objects":7110,"num_object_clones":0,"num_object_copies":21330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7110,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":979,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617841+0000","last_change":"2024-09-14T04:25:28.605221+0000","last_active":"2024-09-14T04:26:36.617841+0000","last_peered":"2024-09-14T04:26:36.617841+0000","last_clean":"2024-09-14T04:26:36.617841+0000","last_became_active":"2024-09-14T04:25:28.604819+0000","last_became_peered":"2024-09-14T04:25:28.604819+0000","last_unstale":"2024-09-14T04:26:36.617841+0000","last_undegraded":"2024-09-14T04:26:36.617841+0000","last_fullsized":"2024-09-14T04:26:36.617841+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:21.033138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":468,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612433+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:26:36.612433+0000","last_peered":"2024-09-14T04:26:36.612433+0000","last_clean":"2024-09-14T04:26:36.612433+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:26:36.612433+0000","last_undegraded":"2024-09-14T04:26:36.612433+0000","last_fullsized":"2024-09-14T04:26:36.612433+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":535,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608226+0000","last_change":"2024-09-14T04:23:27.596032+0000","last_active":"2024-09-14T04:26:36.608226+0000","last_peered":"2024-09-14T04:26:36.608226+0000","last_clean":"2024-09-14T04:26:36.608226+0000","last_became_active":"2024-09-14T04:23:27.595521+0000","last_became_peered":"2024-09-14T04:23:27.595521+0000","last_unstale":"2024-09-14T04:26:36.608226+0000","last_undegraded":"2024-09-14T04:26:36.608226+0000","last_fullsized":"2024-09-14T04:26:36.608226+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:14.145527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":412,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646961+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:26:29.646961+0000","last_peered":"2024-09-14T04:26:29.646961+0000","last_clean":"2024-09-14T04:26:29.646961+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:26:29.646961+0000","last_undegraded":"2024-09-14T04:26:29.646961+0000","last_fullsized":"2024-09-14T04:26:29.646961+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":448,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647319+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:26:29.647319+0000","last_peered":"2024-09-14T04:26:29.647319+0000","last_clean":"2024-09-14T04:26:29.647319+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:26:29.647319+0000","last_undegraded":"2024-09-14T04:26:29.647319+0000","last_fullsized":"2024-09-14T04:26:29.647319+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":447,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647444+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:26:29.647444+0000","last_peered":"2024-09-14T04:26:29.647444+0000","last_clean":"2024-09-14T04:26:29.647444+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:26:29.647444+0000","last_undegraded":"2024-09-14T04:26:29.647444+0000","last_fullsized":"2024-09-14T04:26:29.647444+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617123+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:26:36.617123+0000","last_peered":"2024-09-14T04:26:36.617123+0000","last_clean":"2024-09-14T04:26:36.617123+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:26:36.617123+0000","last_undegraded":"2024-09-14T04:26:36.617123+0000","last_fullsized":"2024-09-14T04:26:36.617123+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":373,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.625385+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:26:36.625385+0000","last_peered":"2024-09-14T04:26:36.625385+0000","last_clean":"2024-09-14T04:26:36.625385+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:26:36.625385+0000","last_undegraded":"2024-09-14T04:26:36.625385+0000","last_fullsized":"2024-09-14T04:26:36.625385+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":914,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613771+0000","last_change":"2024-09-14T04:25:10.214026+0000","last_active":"2024-09-14T04:26:36.613771+0000","last_peered":"2024-09-14T04:26:36.613771+0000","last_clean":"2024-09-14T04:26:36.613771+0000","last_became_active":"2024-09-14T04:25:10.212511+0000","last_became_peered":"2024-09-14T04:25:10.212511+0000","last_unstale":"2024-09-14T04:26:36.613771+0000","last_undegraded":"2024-09-14T04:26:36.613771+0000","last_fullsized":"2024-09-14T04:26:36.613771+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:20.633170+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":319,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:31.086198+0000","last_change":"2024-09-14T04:25:35.559702+0000","last_active":"2024-09-14T04:26:31.086198+0000","last_peered":"2024-09-14T04:26:31.086198+0000","last_clean":"2024-09-14T04:26:31.086198+0000","last_became_active":"2024-09-14T04:25:35.559395+0000","last_became_peered":"2024-09-14T04:25:35.559395+0000","last_unstale":"2024-09-14T04:26:31.086198+0000","last_undegraded":"2024-09-14T04:26:31.086198+0000","last_fullsized":"2024-09-14T04:26:31.086198+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:51.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":309,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:31.086033+0000","last_change":"2024-09-14T04:24:09.122059+0000","last_active":"2024-09-14T04:26:31.086033+0000","last_peered":"2024-09-14T04:26:31.086033+0000","last_clean":"2024-09-14T04:26:31.086033+0000","last_became_active":"2024-09-14T04:24:09.121824+0000","last_became_peered":"2024-09-14T04:24:09.121824+0000","last_unstale":"2024-09-14T04:26:31.086033+0000","last_undegraded":"2024-09-14T04:26:31.086033+0000","last_fullsized":"2024-09-14T04:26:31.086033+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:36.179815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":463,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612012+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:26:36.612012+0000","last_peered":"2024-09-14T04:26:36.612012+0000","last_clean":"2024-09-14T04:26:36.612012+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:26:36.612012+0000","last_undegraded":"2024-09-14T04:26:36.612012+0000","last_fullsized":"2024-09-14T04:26:36.612012+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":567,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609629+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:26:36.609629+0000","last_peered":"2024-09-14T04:26:36.609629+0000","last_clean":"2024-09-14T04:26:36.609629+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:26:36.609629+0000","last_undegraded":"2024-09-14T04:26:36.609629+0000","last_fullsized":"2024-09-14T04:26:36.609629+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644832+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:26:29.644832+0000","last_peered":"2024-09-14T04:26:29.644832+0000","last_clean":"2024-09-14T04:26:29.644832+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:26:29.644832+0000","last_undegraded":"2024-09-14T04:26:29.644832+0000","last_fullsized":"2024-09-14T04:26:29.644832+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643515+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:26:29.643515+0000","last_peered":"2024-09-14T04:26:29.643515+0000","last_clean":"2024-09-14T04:26:29.643515+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:26:29.643515+0000","last_undegraded":"2024-09-14T04:26:29.643515+0000","last_fullsized":"2024-09-14T04:26:29.643515+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":480,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612384+0000","last_change":"2024-09-14T04:25:35.564324+0000","last_active":"2024-09-14T04:26:36.612384+0000","last_peered":"2024-09-14T04:26:36.612384+0000","last_clean":"2024-09-14T04:26:36.612384+0000","last_became_active":"2024-09-14T04:25:35.562918+0000","last_became_peered":"2024-09-14T04:25:35.562918+0000","last_unstale":"2024-09-14T04:26:36.612384+0000","last_undegraded":"2024-09-14T04:26:36.612384+0000","last_fullsized":"2024-09-14T04:26:36.612384+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:16.172185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":727,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613575+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:26:36.613575+0000","last_peered":"2024-09-14T04:26:36.613575+0000","last_clean":"2024-09-14T04:26:36.613575+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:26:36.613575+0000","last_undegraded":"2024-09-14T04:26:36.613575+0000","last_fullsized":"2024-09-14T04:26:36.613575+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":277,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091862+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:26:36.091862+0000","last_peered":"2024-09-14T04:26:36.091862+0000","last_clean":"2024-09-14T04:26:36.091862+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:26:36.091862+0000","last_undegraded":"2024-09-14T04:26:36.091862+0000","last_fullsized":"2024-09-14T04:26:36.091862+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646729+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:26:29.646729+0000","last_peered":"2024-09-14T04:26:29.646729+0000","last_clean":"2024-09-14T04:26:29.646729+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:26:29.646729+0000","last_undegraded":"2024-09-14T04:26:29.646729+0000","last_fullsized":"2024-09-14T04:26:29.646729+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":139,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613901+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:26:36.613901+0000","last_peered":"2024-09-14T04:26:36.613901+0000","last_clean":"2024-09-14T04:26:36.613901+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:26:36.613901+0000","last_undegraded":"2024-09-14T04:26:36.613901+0000","last_fullsized":"2024-09-14T04:26:36.613901+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":477,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610814+0000","last_change":"2024-09-14T04:23:27.573928+0000","last_active":"2024-09-14T04:26:36.610814+0000","last_peered":"2024-09-14T04:26:36.610814+0000","last_clean":"2024-09-14T04:26:36.610814+0000","last_became_active":"2024-09-14T04:23:27.573557+0000","last_became_peered":"2024-09-14T04:23:27.573557+0000","last_unstale":"2024-09-14T04:26:36.610814+0000","last_undegraded":"2024-09-14T04:26:36.610814+0000","last_fullsized":"2024-09-14T04:26:36.610814+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:07.509590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646983+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:26:29.646983+0000","last_peered":"2024-09-14T04:26:29.646983+0000","last_clean":"2024-09-14T04:26:29.646983+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:26:29.646983+0000","last_undegraded":"2024-09-14T04:26:29.646983+0000","last_fullsized":"2024-09-14T04:26:29.646983+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":375,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091762+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:26:36.091762+0000","last_peered":"2024-09-14T04:26:36.091762+0000","last_clean":"2024-09-14T04:26:36.091762+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:26:36.091762+0000","last_undegraded":"2024-09-14T04:26:36.091762+0000","last_fullsized":"2024-09-14T04:26:36.091762+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647027+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:26:29.647027+0000","last_peered":"2024-09-14T04:26:29.647027+0000","last_clean":"2024-09-14T04:26:29.647027+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:26:29.647027+0000","last_undegraded":"2024-09-14T04:26:29.647027+0000","last_fullsized":"2024-09-14T04:26:29.647027+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":384,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091841+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:26:36.091841+0000","last_peered":"2024-09-14T04:26:36.091841+0000","last_clean":"2024-09-14T04:26:36.091841+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:26:36.091841+0000","last_undegraded":"2024-09-14T04:26:36.091841+0000","last_fullsized":"2024-09-14T04:26:36.091841+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":616,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608350+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:26:36.608350+0000","last_peered":"2024-09-14T04:26:36.608350+0000","last_clean":"2024-09-14T04:26:36.608350+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:26:36.608350+0000","last_undegraded":"2024-09-14T04:26:36.608350+0000","last_fullsized":"2024-09-14T04:26:36.608350+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":288,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608377+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:26:36.608377+0000","last_peered":"2024-09-14T04:26:36.608377+0000","last_clean":"2024-09-14T04:26:36.608377+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:26:36.608377+0000","last_undegraded":"2024-09-14T04:26:36.608377+0000","last_fullsized":"2024-09-14T04:26:36.608377+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":409,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646709+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:26:29.646709+0000","last_peered":"2024-09-14T04:26:29.646709+0000","last_clean":"2024-09-14T04:26:29.646709+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:26:29.646709+0000","last_undegraded":"2024-09-14T04:26:29.646709+0000","last_fullsized":"2024-09-14T04:26:29.646709+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":511,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612098+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:26:36.612098+0000","last_peered":"2024-09-14T04:26:36.612098+0000","last_clean":"2024-09-14T04:26:36.612098+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:26:36.612098+0000","last_undegraded":"2024-09-14T04:26:36.612098+0000","last_fullsized":"2024-09-14T04:26:36.612098+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":7654670360,"num_objects":116802,"num_object_clones":0,"num_object_copies":350406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116802,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118134,"num_write_kb":7517889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5374,"num_bytes_recovered":352190464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":23196213248,"data_stored":23196205104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92534,"ondisk_log_size":92534,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":12322,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2506969,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201833,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463471,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834144,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5406328,"kb_used_data":5190192,"kb_used_omap":5,"kb_used_meta":216058,"kb_avail":88338824,"statfs":{"total":95995035648,"available":90458955776,"internally_reserved":0,"allocated":5314756608,"data_stored":5311995228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":221243453},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986743,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088008,"kb_used_data":4721472,"kb_used_omap":1,"kb_used_meta":1366526,"kb_avail":87657144,"statfs":{"total":95995035648,"available":89760915456,"internally_reserved":0,"allocated":4834787328,"data_stored":4832013353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1399323078},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748771,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3793900,"kb_used_data":2813668,"kb_used_omap":4,"kb_used_meta":980219,"kb_avail":89951252,"statfs":{"total":95995035648,"available":92110082048,"internally_reserved":0,"allocated":2881196032,"data_stored":2878438288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1003744812},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286992,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4649800,"kb_used_data":4147196,"kb_used_omap":10,"kb_used_meta":502581,"kb_avail":89095352,"statfs":{"total":95995035648,"available":91233640448,"internally_reserved":0,"allocated":4246728704,"data_stored":4243960030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":514643543},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468315,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3850292,"kb_used_data":2835308,"kb_used_omap":5,"kb_used_meta":1014842,"kb_avail":89894860,"statfs":{"total":95995035648,"available":92052336640,"internally_reserved":0,"allocated":2903355392,"data_stored":2900595831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1039198559},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342430,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3674576,"kb_used_data":3017156,"kb_used_omap":10,"kb_used_meta":657397,"kb_avail":90070576,"statfs":{"total":95995035648,"available":92232269824,"internally_reserved":0,"allocated":3089567744,"data_stored":3086790465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":673174738},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3084320768,"data_stored":3084320768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2877755392,"data_stored":2877751320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4260102144,"data_stored":4260102144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2887254016,"data_stored":2887254016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4817817600,"data_stored":4817813528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5268963328,"data_stored":5268963328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:26:42.796 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:26:42.796 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:26:42.796 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2024-09-14T04:26:42.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:26:42.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:26:42.875 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:42 smithi001 ceph-mon[107399]: pgmap v1421: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 57 MiB/s wr, 983 op/s 2024-09-14T04:26:42.875 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:42 smithi001 ceph-mon[112165]: pgmap v1421: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 57 MiB/s wr, 983 op/s 2024-09-14T04:26:42.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:42.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:42.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:43.055 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:43.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:42 smithi012 ceph-mon[101180]: pgmap v1421: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 57 MiB/s wr, 983 op/s 2024-09-14T04:26:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:43.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:43.214 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:43.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:43.282 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:43.371 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:43.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:43.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:43.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:43.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:43.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:43.705 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:43.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:43 smithi001 ceph-mon[107399]: from='client.46172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:43.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:43 smithi001 ceph-mon[112165]: from='client.46172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:43.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:43.832 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:43.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:43.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:43.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:44.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:44.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:43 smithi012 ceph-mon[101180]: from='client.46172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:44.104 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:44.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:44.245 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:44.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:44.388 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:44.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:44.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:44.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:44.660 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:44.702 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:44.703 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:44.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:44.827 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:44.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:44 smithi001 ceph-mon[112165]: pgmap v1422: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.00k op/s 2024-09-14T04:26:44.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:44 smithi001 ceph-mon[107399]: pgmap v1422: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.00k op/s 2024-09-14T04:26:44.929 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:44.971 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:44.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:45.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:45.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:44 smithi012 ceph-mon[101180]: pgmap v1422: 109 pgs: 109 active+clean; 7.1 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.00k op/s 2024-09-14T04:26:45.099 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:45.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:45.240 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:45.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:45.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:45.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:45.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:45.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:45.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:45.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:45.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:45.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:45.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:45.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:46.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:46.062 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:46.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:46.201 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:46.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:46.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:46.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:46.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.464 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:46.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:46.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:46.599 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:46.700 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:46.739 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:46.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:46.872 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:46.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:46.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:46 smithi001 ceph-mon[112165]: pgmap v1423: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 44 MiB/s wr, 1.02k op/s 2024-09-14T04:26:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:46 smithi001 ceph-mon[107399]: pgmap v1423: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 44 MiB/s wr, 1.02k op/s 2024-09-14T04:26:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:46] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.51.0" 2024-09-14T04:26:46.973 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:47.009 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:46 smithi012 ceph-mon[101180]: pgmap v1423: 109 pgs: 109 active+clean; 7.0 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 44 MiB/s wr, 1.02k op/s 2024-09-14T04:26:47.110 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:47.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:47.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:47.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:47.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:47.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:47.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:47.538 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:47.575 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:47.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:47.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:47.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:47.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:47.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:47.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:48.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:48.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:48.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:48.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:48.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:48.395 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:48.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145635 12.8 12:180d0589:::benchmark_data_smithi001_255951_object145634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 421.966222426, 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5285736509999999, 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.527 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145635 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:26:48.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.529 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.529 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.529 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430910+0000", 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430911+0000", 2024-09-14T04:26:48.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.593e-06 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430936+0000", 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4176999999999999e-05 2024-09-14T04:26:48.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430949+0000", 2024-09-14T04:26:48.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3064000000000001e-05 2024-09-14T04:26:48.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.430957+0000", 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0010000000000001e-06 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431014+0000", 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7336999999999997e-05 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431148+0000", 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013379700000000001 2024-09-14T04:26:48.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.431286+0000", 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013852299999999999 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.439229+0000", 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0079427400000000002 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457555+0000", 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018325807 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457568+0000", 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2772000000000001e-05 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.447670+0000", 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99010279800000001 2024-09-14T04:26:48.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.447682+0000", 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2094e-05 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959396+0000", 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.51171327 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959483+0000", 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7677999999999996e-05 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145637 12.8 12:1ca95b6c:::benchmark_data_smithi001_255951_object145636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 421.96444841800002, 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5270983039999999, 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.539 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145637 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432684+0000", 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432685+0000", 2024-09-14T04:26:48.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5689999999999999e-06 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432710+0000", 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5086e-05 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432728+0000", 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7439999999999999e-05 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.432735+0000", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9890000000000003e-06 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435266+0000", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025307289999999998 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435427+0000", 2024-09-14T04:26:48.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000161547 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0", 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.435604+0000", 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000176935 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.442032+0000", 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064282209999999996 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457611+0000", 2024-09-14T04:26:48.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015578843 2024-09-14T04:26:48.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.457622+0000", 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0392e-05 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448245+0000", 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99062326499999998 2024-09-14T04:26:48.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448256+0000", 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.112e-05 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959734+0000", 2024-09-14T04:26:48.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5114783090000001 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959782+0000", 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7858999999999997e-05 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145649 12.8 12:13d74b24:::benchmark_data_smithi001_255951_object145648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:26:48.550 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 421.935997235, 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.505730679, 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145649 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461135+0000", 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461136+0000", 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5090000000000001e-06 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461161+0000", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4749999999999999e-05 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461177+0000", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6164e-05 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461187+0000", 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5200000000000003e-06 2024-09-14T04:26:48.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461244+0000", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6696000000000001e-05 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.461253+0000", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0140000000000004e-06 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448292+0000", 2024-09-14T04:26:48.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98703899100000003 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448300+0000", 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9910000000000002e-06 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959840+0000", 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.511540222 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.959989+0000", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014936699999999999 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.960181+0000", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019227099999999999 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966617+0000", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0064353800000000001 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966635+0000", 2024-09-14T04:26:48.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.844e-05 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966682+0000", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6220000000000001e-05 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966753+0000", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1348999999999997e-05 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966866+0000", 2024-09-14T04:26:48.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011279499999999999 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145657 12.8 12:110bae6a:::benchmark_data_smithi001_255951_object145656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 421.92347647100001, 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4938081240000001, 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.558 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145657 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473656+0000", 2024-09-14T04:26:48.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473658+0000", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8220000000000001e-06 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473682+0000", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4878000000000001e-05 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473698+0000", 2024-09-14T04:26:48.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5522000000000001e-05 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473707+0000", 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1530000000000003e-06 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473768+0000", 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0510000000000002e-05 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.473779+0000", 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1559999999999999e-05 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448389+0000", 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97460960100000005 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.448397+0000", 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3790000000000006e-06 2024-09-14T04:26:48.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963298+0000", 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5149011990000001 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963504+0000", 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020578299999999999 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:26:48.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.963697+0000", 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000192475 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966883+0000", 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031863590000000002 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.966911+0000", 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7776999999999998e-05 2024-09-14T04:26:48.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967313+0000", 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00040244800000000002 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967370+0000", 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6876000000000002e-05 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.967464+0000", 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3782000000000004e-05 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145630 12.b 12:d5814912:::benchmark_data_smithi001_255951_object145629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.95774167500002, 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5046016870000001, 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:26:48.566 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145630 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439391+0000", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439392+0000", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.508e-06 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439421+0000", 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.8538e-05 2024-09-14T04:26:48.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439438+0000", 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7567999999999998e-05 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439450+0000", 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1590000000000001e-05 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439594+0000", 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000144671 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439604+0000", 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7070000000000004e-06 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444053+0000", 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004448708 2024-09-14T04:26:48.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444061+0000", 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3489999999999992e-06 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.941337+0000", 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4972761270000001 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.943992+0000", 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026549210000000002 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145648 12.b 12:de9db7e2:::benchmark_data_smithi001_255951_object145647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 420.95766128000002, 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5172668740000002, 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.572 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145648 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439471+0000", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439472+0000", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7800000000000001e-07 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439523+0000", 2024-09-14T04:26:48.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1477000000000002e-05 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439533+0000", 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0023999999999999e-05 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439540+0000", 2024-09-14T04:26:48.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8000000000000001e-06 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439720+0000", 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000180237 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.439726+0000", 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1160000000000004e-06 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444090+0000", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043639409999999997 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.444097+0000", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5189999999999998e-06 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944233+0000", 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.500135888 2024-09-14T04:26:48.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.945064+0000", 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00083095700000000001 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.945586+0000", 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00052237900000000001 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.955563+0000", 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099768319999999997 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956014+0000", 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00045092500000000002 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956568+0000", 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055460500000000001 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956625+0000", 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6283000000000003e-05 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.956738+0000", 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113213 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:147029 12.1 12:84f6be8d:::benchmark_data_smithi001_255951_object1369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 417.32199619800002, 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22532575899999999, 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 147029 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075136+0000", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075138+0000", 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7039999999999999e-06 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075143+0000", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3260000000000003e-06 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075159+0000", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5775999999999999e-05 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075166+0000", 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3459999999999996e-06 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075219+0000", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3031000000000001e-05 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075329+0000", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011004400000000001 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.075443+0000", 2024-09-14T04:26:48.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113313 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.078989+0000", 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035467020000000001 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.079083+0000", 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3555999999999997e-05 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300371+0000", 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22128830699999999 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300412+0000", 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0707000000000002e-05 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:51.300462+0000", 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9947000000000001e-05 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.586 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:167946 12.b 12:da82cc1e:::benchmark_data_smithi001_255951_object22286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 393.26039132099999, 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21001428599999999, 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.587 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 167946 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136741+0000", 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136743+0000", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.697e-06 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136749+0000", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9680000000000003e-06 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136764+0000", 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5807e-05 2024-09-14T04:26:48.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136772+0000", 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0229999999999992e-06 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136841+0000", 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8594000000000001e-05 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.590 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.136996+0000", 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015524000000000001 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.137130+0000", 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013398199999999999 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.142972+0000", 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005841499 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.143744+0000", 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00077236600000000002 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346552+0000", 2024-09-14T04:26:48.592 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20280820299999999 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346627+0000", 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4492000000000004e-05 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:15.346755+0000", 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000128415 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.593 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:182473 12.1 12:8e0dbc1e:::benchmark_data_smithi001_255951_object36813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 376.10628590499999, 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.237500503, 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.594 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182473 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290846+0000", 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290848+0000", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.702e-06 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290854+0000", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8989999999999998e-06 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290870+0000", 2024-09-14T04:26:48.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6319000000000001e-05 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290880+0000", 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5380000000000008e-06 2024-09-14T04:26:48.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.290948+0000", 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8319999999999996e-05 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.291075+0000", 2024-09-14T04:26:48.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012682599999999999 2024-09-14T04:26:48.599 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.599 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:26:48.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.291214+0000", 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013861200000000001 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.294980+0000", 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037662669999999998 2024-09-14T04:26:48.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.295104+0000", 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000123707 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528130+0000", 2024-09-14T04:26:48.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23302693799999999 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528297+0000", 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016663499999999999 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.602 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528347+0000", 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9740000000000001e-05 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:182478 12.1 12:86b6932c:::benchmark_data_smithi001_255951_object36818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-14T04:26:48.603 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 376.09753929700003, 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22889717000000001, 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 182478 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.604 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299593+0000", 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.605 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299594+0000", 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.55e-06 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299599+0000", 2024-09-14T04:26:48.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9259999999999999e-06 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299613+0000", 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3648e-05 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299620+0000", 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0820000000000001e-06 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299674+0000", 2024-09-14T04:26:48.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3477999999999999e-05 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299751+0000", 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7391999999999994e-05 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.299818+0000", 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7144999999999995e-05 2024-09-14T04:26:48.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.302548+0000", 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002730105 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.302679+0000", 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013062999999999999 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528395+0000", 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22571648499999999 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528451+0000", 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5522000000000001e-05 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:20:32.528490+0000", 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9206999999999999e-05 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.611 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246146 12.1 12:86d285cb:::benchmark_data_smithi001_255951_object100486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:26:48.612 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 257.59303685100002, 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.70537081599999996, 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.613 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246146 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:26:48.614 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804097+0000", 2024-09-14T04:26:48.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2139999999999999e-06 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804102+0000", 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1560000000000003e-06 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804115+0000", 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3072e-05 2024-09-14T04:26:48.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804122+0000", 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.849e-06 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804175+0000", 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3134000000000003e-05 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804180+0000", 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5799999999999999e-06 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500297+0000", 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.69611708299999997 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500422+0000", 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000124814 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500539+0000", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011681299999999999 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503578+0000", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00303915 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503636+0000", 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7541999999999999e-05 2024-09-14T04:26:48.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509290+0000", 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056538700000000001 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509357+0000", 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6892999999999998e-05 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509466+0000", 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000109646 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246149 12.1 12:815a4314:::benchmark_data_smithi001_255951_object100489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 257.51695024399999, 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62963798900000001, 2024-09-14T04:26:48.621 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246149 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.622 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880184+0000", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8500000000000001e-06 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880191+0000", 2024-09-14T04:26:48.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4139999999999997e-06 2024-09-14T04:26:48.624 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880206+0000", 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5213e-05 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880214+0000", 2024-09-14T04:26:48.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8699999999999992e-06 2024-09-14T04:26:48.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880277+0000", 2024-09-14T04:26:48.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2564999999999995e-05 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880289+0000", 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1912e-05 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503667+0000", 2024-09-14T04:26:48.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62337792000000003 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503790+0000", 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012371800000000001 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503926+0000", 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135828 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509269+0000", 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005342935 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509528+0000", 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025847700000000001 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509684+0000", 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015632800000000001 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509732+0000", 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7926000000000002e-05 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509820+0000", 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8033000000000002e-05 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.630 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246168 12.1 12:80e4f094:::benchmark_data_smithi001_255951_object100508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 257.36831790500003, 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.48125710599999999, 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246168 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.631 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028815+0000", 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.15e-06 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028820+0000", 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.51e-06 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028839+0000", 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9276999999999999e-05 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.633 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028844+0000", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8930000000000004e-06 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028906+0000", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1419000000000003e-05 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028917+0000", 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1065e-05 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506507+0000", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.47759034299999997 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506620+0000", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011265299999999999 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506740+0000", 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120423 2024-09-14T04:26:48.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509580+0000", 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002840371 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509849+0000", 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00026879699999999998 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509915+0000", 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6259000000000004e-05 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509976+0000", 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0838999999999998e-05 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510071+0000", 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5106999999999996e-05 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.637 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246175 12.1 12:8f83bc3f:::benchmark_data_smithi001_255951_object100515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 257.30688917700002, 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.42337427, 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.638 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246175 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090245+0000", 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.562e-06 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090251+0000", 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1940000000000003e-06 2024-09-14T04:26:48.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090265+0000", 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4567e-05 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090274+0000", 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6070000000000008e-06 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090339+0000", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5283999999999994e-05 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090346+0000", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6109999999999999e-06 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510119+0000", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41977286000000003 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.642 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510221+0000", 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102538 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510288+0000", 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6341999999999996e-05 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513037+0000", 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027498499999999999 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513154+0000", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116457 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513461+0000", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030725199999999999 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513523+0000", 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1754999999999994e-05 2024-09-14T04:26:48.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513617+0000", 2024-09-14T04:26:48.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4390999999999998e-05 2024-09-14T04:26:48.645 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:274519 12.11 12:8e2b50db:::benchmark_data_smithi001_255951_object128859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-14T04:26:48.646 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:42.680486+0000", 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 185.71664579700001, 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.236567792, 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 274519 2024-09-14T04:26:48.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680486+0000", 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680486+0000", 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680488+0000", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.891e-06 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680495+0000", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1890000000000004e-06 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680507+0000", 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2165e-05 2024-09-14T04:26:48.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680514+0000", 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0419999999999997e-06 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680579+0000", 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5636000000000007e-05 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.680719+0000", 2024-09-14T04:26:48.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013925100000000001 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2,7", 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.681030+0000", 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000311012 2024-09-14T04:26:48.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.682577+0000", 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001547252 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.682874+0000", 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00029716999999999998 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.654 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683291+0000", 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000417413 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.655 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.916873+0000", 2024-09-14T04:26:48.656 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.233581122 2024-09-14T04:26:48.656 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.656 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.656 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.656 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.916940+0000", 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7102999999999998e-05 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917054+0000", 2024-09-14T04:26:48.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114546 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.658 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:274520 12.11 12:8b046b61:::benchmark_data_smithi001_255951_object128860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:42.683181+0000", 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 185.71395163400001, 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23407072100000001, 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.659 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 274520 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.660 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683181+0000", 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.661 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.662 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683181+0000", 2024-09-14T04:26:48.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683182+0000", 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.198e-06 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683187+0000", 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6910000000000002e-06 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683208+0000", 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0392000000000001e-05 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683215+0000", 2024-09-14T04:26:48.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.593e-06 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683319+0000", 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000103546 2024-09-14T04:26:48.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683458+0000", 2024-09-14T04:26:48.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000139369 2024-09-14T04:26:48.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2,7", 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.683727+0000", 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00026855 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.685134+0000", 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014074700000000001 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.669 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.685158+0000", 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3901999999999999e-05 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.685701+0000", 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054249599999999995 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917096+0000", 2024-09-14T04:26:48.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.231395347 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917146+0000", 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0003000000000002e-05 2024-09-14T04:26:48.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917251+0000", 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105164 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.673 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.674 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.674 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:274530 12.1 12:87a8b534:::benchmark_data_smithi001_255951_object128870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-14T04:26:48.674 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:42.700698+0000", 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 185.69643385099999, 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.217207436, 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.675 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 274530 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700698+0000", 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700698+0000", 2024-09-14T04:26:48.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700700+0000", 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9460000000000001e-06 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700706+0000", 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1040000000000003e-06 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700722+0000", 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5341000000000001e-05 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700731+0000", 2024-09-14T04:26:48.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2790000000000004e-06 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700794+0000", 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2550000000000003e-05 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.700945+0000", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015144600000000001 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,2,7", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.701298+0000", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035332000000000001 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.702759+0000", 2024-09-14T04:26:48.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001460575 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.703097+0000", 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00033833499999999999 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.703150+0000", 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2571999999999997e-05 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917719+0000", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.21456921500000001 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917780+0000", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0690000000000003e-05 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:42.917906+0000", 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012606299999999999 2024-09-14T04:26:48.683 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.46076.0:82499 13.2 e423/414 13:4ee5e20b:::benchmark_data_smithi001_306460_object82498:head v 423'5202, mlcod=406'1423)", 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:04.167273+0000", 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 44.229858907000001, 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.059950829999999997, 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167273+0000", 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167273+0000", 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167274+0000", 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3099999999999997e-07 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167363+0000", 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8922e-05 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167377+0000", 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4433999999999999e-05 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.167385+0000", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.515e-06 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.177027+0000", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0096416369999999998 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.177113+0000", 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6347000000000005e-05 2024-09-14T04:26:48.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.227036+0000", 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.049923370000000002 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.227224+0000", 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018787400000000001 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.46076.0:82509 13.2 e423/414 13:4fc7623c:::benchmark_data_smithi001_306460_object82508:head v 423'5203, mlcod=406'1423)", 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:04.175795+0000", 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 44.221337048999999, 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.051583100999999999, 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175795+0000", 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175795+0000", 2024-09-14T04:26:48.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175796+0000", 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.018e-06 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175841+0000", 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5309999999999998e-05 2024-09-14T04:26:48.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175854+0000", 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2849e-05 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.175862+0000", 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4259999999999997e-06 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.182134+0000", 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062724410000000001 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.182188+0000", 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4008999999999997e-05 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.227229+0000", 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.045040496999999999 2024-09-14T04:26:48.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:04.227378+0000", 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014955099999999999 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.46076.0:93689 13.d e424/398 13:beb8846c:::benchmark_data_smithi001_306460_object93688:head v 424'5997, mlcod=424'5995)", 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:15.018942+0000", 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 33.378190154999999, 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.052546102999999997, 2024-09-14T04:26:48.694 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.018942+0000", 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.018942+0000", 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:26:48.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.018943+0000", 2024-09-14T04:26:48.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5799999999999999e-07 2024-09-14T04:26:48.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.018981+0000", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8130000000000003e-05 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.062387+0000", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043405661999999998 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.062399+0000", 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2825e-05 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.062559+0000", 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015918 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.062580+0000", 2024-09-14T04:26:48.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1328e-05 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.071439+0000", 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088594160000000002 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:15.071488+0000", 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8903999999999998e-05 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.700 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.701 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:26:48.701 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:26:48.701 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:26:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:48.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:48.832 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:48.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:48.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:48.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:48 smithi001 ceph-mon[112165]: pgmap v1424: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 93 B/s rd, 30 MiB/s wr, 1.12k op/s 2024-09-14T04:26:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:48 smithi001 ceph-mon[107399]: pgmap v1424: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 93 B/s rd, 30 MiB/s wr, 1.12k op/s 2024-09-14T04:26:49.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:49.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:48 smithi012 ceph-mon[101180]: pgmap v1424: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 93 B/s rd, 30 MiB/s wr, 1.12k op/s 2024-09-14T04:26:49.104 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:49.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.249 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:49.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:49.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:49.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:49.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:49.509 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:49.610 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:49.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:49.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:49.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:49.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:49.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:49.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:49.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:50.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:50.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:50.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.187 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:50.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:50.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:50.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:50.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:50.465 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:50.567 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:50.604 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:50.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:50.746 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:50.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:50.884 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:50.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:50 smithi001 ceph-mon[112165]: pgmap v1425: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 892 op/s 2024-09-14T04:26:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:50 smithi001 ceph-mon[107399]: pgmap v1425: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 892 op/s 2024-09-14T04:26:50.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:50.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:51.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:51.073 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:50 smithi012 ceph-mon[101180]: pgmap v1425: 109 pgs: 109 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 892 op/s 2024-09-14T04:26:51.073 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:51.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:51.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:51.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.308 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:51.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:51.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.445 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:51.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:51.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:51.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:51.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:51.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:51.724 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:51.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:51.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:51.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:51.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:51.994 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:51.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:52.095 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:52.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:52.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:52.261 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:52.363 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:52.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:52.505 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:52.551 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:52.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:52.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:52.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:52.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:52 smithi001 ceph-mon[112165]: pgmap v1426: 109 pgs: 109 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 1.23k op/s 2024-09-14T04:26:52.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:52 smithi001 ceph-mon[107399]: pgmap v1426: 109 pgs: 109 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 1.23k op/s 2024-09-14T04:26:52.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:26:52.924 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:52 smithi012 ceph-mon[101180]: pgmap v1426: 109 pgs: 109 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 19 MiB/s wr, 1.23k op/s 2024-09-14T04:26:52.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:52.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:52.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:52.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:53.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:53.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:53.206 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:26:53.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:53.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:53.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:53.393 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:53.494 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:53.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:53.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:53.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:53.654 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:26:53.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:53.757 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2642852508' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:53.757 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:53.758 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2642852508' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:53.758 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:53.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:53.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:53.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:53.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:53.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:54.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.071 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:54.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2642852508' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:26:54.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:54.194 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:26:54.196 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:26:54.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:54.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:54.339 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:54.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:54.474 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:54.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:54.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:54.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:54.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:54.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:54.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:54.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: pgmap v1427: 109 pgs: 109 active+clean; 6.4 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 4.0 MiB/s wr, 1.09k op/s 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: osdmap e428: 8 total, 8 up, 6 in 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3606969680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:54.888 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:54 smithi012 ceph-mon[101180]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: pgmap v1427: 109 pgs: 109 active+clean; 6.4 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 4.0 MiB/s wr, 1.09k op/s 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: osdmap e428: 8 total, 8 up, 6 in 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3606969680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[107399]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: pgmap v1427: 109 pgs: 109 active+clean; 6.4 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 4.0 MiB/s wr, 1.09k op/s 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:26:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: osdmap e428: 8 total, 8 up, 6 in 2024-09-14T04:26:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3606969680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:26:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:54 smithi001 ceph-mon[112165]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:26:54.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:55.017 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:55.119 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:55.151 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:55.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:55.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:55.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:55.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:55.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:55.551 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:55.637 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:26:55.638 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:26:55.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:55.686 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:55.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:55.815 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:55.857 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:26:55.858 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:26:55.859 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:26:55.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:55.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:55.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:55.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:55 smithi001 ceph-mon[112165]: pgmap v1429: 109 pgs: 109 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:55 smithi001 ceph-mon[107399]: pgmap v1429: 109 pgs: 109 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:56.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:56.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:55 smithi012 ceph-mon[101180]: pgmap v1429: 109 pgs: 109 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:56.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:56.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:56.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:56.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:56.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:56.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:56.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:56.493 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:56.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:26:56.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:56.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:56.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:56.774 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:56.834 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:26:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:26:56] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-09-14T04:26:56.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:56.886 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:26:56.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:56.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:57.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:57.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:57.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:57.189 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:57.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:57.326 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:57.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:26:57.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:26:57.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:26:57.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:57.732 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:57.834 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:57.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:57 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:26:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:57 smithi012 ceph-mon[101180]: 1.e scrub ok 2024-09-14T04:26:57.870 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:57.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:57.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:57 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:26:57.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:57 smithi001 ceph-mon[107399]: 1.e scrub ok 2024-09-14T04:26:57.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:57 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:26:57.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:57 smithi001 ceph-mon[112165]: 1.e scrub ok 2024-09-14T04:26:57.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:58.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:58.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:58.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:26:58.296 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:26:58.398 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:58.438 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:58.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:58.596 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1428,"stamp":"2024-09-14T04:26:53.648582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6897369296,"num_objects":105263,"num_object_clones":0,"num_object_copies":315789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105263,"num_whiteouts":0,"num_read":18,"num_read_kb":39,"num_write":131321,"num_write_kb":7567568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5441,"num_bytes_recovered":354709755,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95735,"ondisk_log_size":95735,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":25373760,"kb_used_data":20585616,"kb_used_omap":38,"kb_used_meta":4787737,"kb_avail":537097152,"statfs":{"total":575970213888,"available":549987483648,"internally_reserved":0,"allocated":21079670784,"data_stored":21063064245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39753,"internal_metadata":4902642871},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,1,13,5,6,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-753074176,"num_objects":-11491,"num_object_clones":0,"num_object_copies":-34473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-11491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11529,"num_write_kb":1216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.130618"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":657,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.606756+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:26:36.606756+0000","last_peered":"2024-09-14T04:26:36.606756+0000","last_clean":"2024-09-14T04:26:36.606756+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:26:36.606756+0000","last_undegraded":"2024-09-14T04:26:36.606756+0000","last_fullsized":"2024-09-14T04:26:36.606756+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":163,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618733+0000","last_change":"2024-09-14T04:23:28.577777+0000","last_active":"2024-09-14T04:26:36.618733+0000","last_peered":"2024-09-14T04:26:36.618733+0000","last_clean":"2024-09-14T04:26:36.618733+0000","last_became_active":"2024-09-14T04:23:28.577155+0000","last_became_peered":"2024-09-14T04:23:28.577155+0000","last_unstale":"2024-09-14T04:26:36.618733+0000","last_undegraded":"2024-09-14T04:26:36.618733+0000","last_fullsized":"2024-09-14T04:26:36.618733+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:01.212541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":175,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647445+0000","last_change":"2024-09-14T04:22:11.330347+0000","last_active":"2024-09-14T04:26:29.647445+0000","last_peered":"2024-09-14T04:26:29.647445+0000","last_clean":"2024-09-14T04:26:29.647445+0000","last_became_active":"2024-09-14T04:22:11.329918+0000","last_became_peered":"2024-09-14T04:22:11.329918+0000","last_unstale":"2024-09-14T04:26:29.647445+0000","last_undegraded":"2024-09-14T04:26:29.647445+0000","last_fullsized":"2024-09-14T04:26:29.647445+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:42.872048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":765,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643865+0000","last_change":"2024-09-14T04:24:49.569053+0000","last_active":"2024-09-14T04:26:29.643865+0000","last_peered":"2024-09-14T04:26:29.643865+0000","last_clean":"2024-09-14T04:26:29.643865+0000","last_became_active":"2024-09-14T04:24:49.568690+0000","last_became_peered":"2024-09-14T04:24:49.568690+0000","last_unstale":"2024-09-14T04:26:29.643865+0000","last_undegraded":"2024-09-14T04:26:29.643865+0000","last_fullsized":"2024-09-14T04:26:29.643865+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:19:37.952706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644601+0000","last_change":"2024-09-14T04:25:25.582150+0000","last_active":"2024-09-14T04:26:29.644601+0000","last_peered":"2024-09-14T04:26:29.644601+0000","last_clean":"2024-09-14T04:26:29.644601+0000","last_became_active":"2024-09-14T04:25:25.581683+0000","last_became_peered":"2024-09-14T04:25:25.581683+0000","last_unstale":"2024-09-14T04:26:29.644601+0000","last_undegraded":"2024-09-14T04:26:29.644601+0000","last_fullsized":"2024-09-14T04:26:29.644601+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:10.188214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":473,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.603848+0000","last_change":"2024-09-14T04:25:35.562683+0000","last_active":"2024-09-14T04:26:36.603848+0000","last_peered":"2024-09-14T04:26:36.603848+0000","last_clean":"2024-09-14T04:26:36.603848+0000","last_became_active":"2024-09-14T04:25:35.562281+0000","last_became_peered":"2024-09-14T04:25:35.562281+0000","last_unstale":"2024-09-14T04:26:36.603848+0000","last_undegraded":"2024-09-14T04:26:36.603848+0000","last_fullsized":"2024-09-14T04:26:36.603848+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:09.493757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":947,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614133+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:26:36.614133+0000","last_peered":"2024-09-14T04:26:36.614133+0000","last_clean":"2024-09-14T04:26:36.614133+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:26:36.614133+0000","last_undegraded":"2024-09-14T04:26:36.614133+0000","last_fullsized":"2024-09-14T04:26:36.614133+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":459,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612527+0000","last_change":"2024-09-14T04:22:11.337650+0000","last_active":"2024-09-14T04:26:36.612527+0000","last_peered":"2024-09-14T04:26:36.612527+0000","last_clean":"2024-09-14T04:26:36.612527+0000","last_became_active":"2024-09-14T04:22:11.336894+0000","last_became_peered":"2024-09-14T04:22:11.336894+0000","last_unstale":"2024-09-14T04:26:36.612527+0000","last_undegraded":"2024-09-14T04:26:36.612527+0000","last_fullsized":"2024-09-14T04:26:36.612527+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:23.268544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":381,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647100+0000","last_change":"2024-09-14T04:22:11.321475+0000","last_active":"2024-09-14T04:26:29.647100+0000","last_peered":"2024-09-14T04:26:29.647100+0000","last_clean":"2024-09-14T04:26:29.647100+0000","last_became_active":"2024-09-14T04:22:11.321165+0000","last_became_peered":"2024-09-14T04:22:11.321165+0000","last_unstale":"2024-09-14T04:26:29.647100+0000","last_undegraded":"2024-09-14T04:26:29.647100+0000","last_fullsized":"2024-09-14T04:26:29.647100+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:24.507034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":593,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643766+0000","last_change":"2024-09-14T04:24:49.572838+0000","last_active":"2024-09-14T04:26:29.643766+0000","last_peered":"2024-09-14T04:26:29.643766+0000","last_clean":"2024-09-14T04:26:29.643766+0000","last_became_active":"2024-09-14T04:24:49.572388+0000","last_became_peered":"2024-09-14T04:24:49.572388+0000","last_unstale":"2024-09-14T04:26:29.643766+0000","last_undegraded":"2024-09-14T04:26:29.643766+0000","last_fullsized":"2024-09-14T04:26:29.643766+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:22.578056+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617890+0000","last_change":"2024-09-14T04:25:35.565538+0000","last_active":"2024-09-14T04:26:36.617890+0000","last_peered":"2024-09-14T04:26:36.617890+0000","last_clean":"2024-09-14T04:26:36.617890+0000","last_became_active":"2024-09-14T04:25:35.565210+0000","last_became_peered":"2024-09-14T04:25:35.565210+0000","last_unstale":"2024-09-14T04:26:36.617890+0000","last_undegraded":"2024-09-14T04:26:36.617890+0000","last_fullsized":"2024-09-14T04:26:36.617890+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:08.538693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":541,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610085+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:26:36.610085+0000","last_peered":"2024-09-14T04:26:36.610085+0000","last_clean":"2024-09-14T04:26:36.610085+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:26:36.610085+0000","last_undegraded":"2024-09-14T04:26:36.610085+0000","last_fullsized":"2024-09-14T04:26:36.610085+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"427'8072","reported_seq":8141,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:52.304004+0000","last_change":"2024-09-14T04:25:02.146250+0000","last_active":"2024-09-14T04:26:52.304004+0000","last_peered":"2024-09-14T04:26:52.304004+0000","last_clean":"2024-09-14T04:26:52.304004+0000","last_became_active":"2024-09-14T04:25:02.145730+0000","last_became_peered":"2024-09-14T04:25:02.145730+0000","last_unstale":"2024-09-14T04:26:52.304004+0000","last_undegraded":"2024-09-14T04:26:52.304004+0000","last_fullsized":"2024-09-14T04:26:52.304004+0000","mapping_epoch":398,"log_start":"406'1500","ondisk_log_start":"406'1500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:04.224581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411828224,"num_objects":6284,"num_object_clones":0,"num_object_copies":18852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8072,"num_write_kb":459392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":749,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647690+0000","last_change":"2024-09-14T04:24:49.572850+0000","last_active":"2024-09-14T04:26:29.647690+0000","last_peered":"2024-09-14T04:26:29.647690+0000","last_clean":"2024-09-14T04:26:29.647690+0000","last_became_active":"2024-09-14T04:24:49.572405+0000","last_became_peered":"2024-09-14T04:24:49.572405+0000","last_unstale":"2024-09-14T04:26:29.647690+0000","last_undegraded":"2024-09-14T04:26:29.647690+0000","last_fullsized":"2024-09-14T04:26:29.647690+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:06.709118+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":144,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.603722+0000","last_change":"2024-09-14T04:23:28.575634+0000","last_active":"2024-09-14T04:26:36.603722+0000","last_peered":"2024-09-14T04:26:36.603722+0000","last_clean":"2024-09-14T04:26:36.603722+0000","last_became_active":"2024-09-14T04:23:28.558391+0000","last_became_peered":"2024-09-14T04:23:28.558391+0000","last_unstale":"2024-09-14T04:26:36.603722+0000","last_undegraded":"2024-09-14T04:26:36.603722+0000","last_fullsized":"2024-09-14T04:26:36.603722+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:03.845925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643649+0000","last_change":"2024-09-14T04:25:52.595074+0000","last_active":"2024-09-14T04:26:29.643649+0000","last_peered":"2024-09-14T04:26:29.643649+0000","last_clean":"2024-09-14T04:26:29.643649+0000","last_became_active":"2024-09-14T04:25:52.594744+0000","last_became_peered":"2024-09-14T04:25:52.594744+0000","last_unstale":"2024-09-14T04:26:29.643649+0000","last_undegraded":"2024-09-14T04:26:29.643649+0000","last_fullsized":"2024-09-14T04:26:29.643649+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:10.267321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"427'8225","reported_seq":8294,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.095213+0000","last_change":"2024-09-14T04:25:01.992464+0000","last_active":"2024-09-14T04:26:51.095213+0000","last_peered":"2024-09-14T04:26:51.095213+0000","last_clean":"2024-09-14T04:26:51.095213+0000","last_became_active":"2024-09-14T04:25:01.992117+0000","last_became_peered":"2024-09-14T04:25:01.992117+0000","last_unstale":"2024-09-14T04:26:51.095213+0000","last_undegraded":"2024-09-14T04:26:51.095213+0000","last_fullsized":"2024-09-14T04:26:51.095213+0000","mapping_epoch":398,"log_start":"412'3000","ondisk_log_start":"412'3000","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:21.289671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431554560,"num_objects":6585,"num_object_clones":0,"num_object_copies":19755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8225,"num_write_kb":473920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1000,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617877+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:26:36.617877+0000","last_peered":"2024-09-14T04:26:36.617877+0000","last_clean":"2024-09-14T04:26:36.617877+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:26:36.617877+0000","last_undegraded":"2024-09-14T04:26:36.617877+0000","last_fullsized":"2024-09-14T04:26:36.617877+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":40,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646979+0000","last_change":"2024-09-14T04:25:07.963635+0000","last_active":"2024-09-14T04:26:29.646979+0000","last_peered":"2024-09-14T04:26:29.646979+0000","last_clean":"2024-09-14T04:26:29.646979+0000","last_became_active":"2024-09-14T04:25:07.959540+0000","last_became_peered":"2024-09-14T04:25:07.959540+0000","last_unstale":"2024-09-14T04:26:29.646979+0000","last_undegraded":"2024-09-14T04:26:29.646979+0000","last_fullsized":"2024-09-14T04:26:29.646979+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:05.353685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":453,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646861+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:26:29.646861+0000","last_peered":"2024-09-14T04:26:29.646861+0000","last_clean":"2024-09-14T04:26:29.646861+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:26:29.646861+0000","last_undegraded":"2024-09-14T04:26:29.646861+0000","last_fullsized":"2024-09-14T04:26:29.646861+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"427'8138","reported_seq":8205,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:49.562282+0000","last_change":"2024-09-14T04:25:02.146419+0000","last_active":"2024-09-14T04:26:49.562282+0000","last_peered":"2024-09-14T04:26:49.562282+0000","last_clean":"2024-09-14T04:26:49.562282+0000","last_became_active":"2024-09-14T04:25:02.145877+0000","last_became_peered":"2024-09-14T04:25:02.145877+0000","last_unstale":"2024-09-14T04:26:49.562282+0000","last_undegraded":"2024-09-14T04:26:49.562282+0000","last_fullsized":"2024-09-14T04:26:49.562282+0000","mapping_epoch":398,"log_start":"412'2600","ondisk_log_start":"412'2600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:31.774348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442105856,"num_objects":6746,"num_object_clones":0,"num_object_copies":20238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8138,"num_write_kb":476288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":449,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644950+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:26:29.644950+0000","last_peered":"2024-09-14T04:26:29.644950+0000","last_clean":"2024-09-14T04:26:29.644950+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:26:29.644950+0000","last_undegraded":"2024-09-14T04:26:29.644950+0000","last_fullsized":"2024-09-14T04:26:29.644950+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":34,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644485+0000","last_change":"2024-09-14T04:25:25.584570+0000","last_active":"2024-09-14T04:26:29.644485+0000","last_peered":"2024-09-14T04:26:29.644485+0000","last_clean":"2024-09-14T04:26:29.644485+0000","last_became_active":"2024-09-14T04:25:25.581721+0000","last_became_peered":"2024-09-14T04:25:25.581721+0000","last_unstale":"2024-09-14T04:26:29.644485+0000","last_undegraded":"2024-09-14T04:26:29.644485+0000","last_fullsized":"2024-09-14T04:26:29.644485+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:59.618866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":446,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617461+0000","last_change":"2024-09-14T04:24:08.985773+0000","last_active":"2024-09-14T04:26:36.617461+0000","last_peered":"2024-09-14T04:26:36.617461+0000","last_clean":"2024-09-14T04:26:36.617461+0000","last_became_active":"2024-09-14T04:24:08.985458+0000","last_became_peered":"2024-09-14T04:24:08.985458+0000","last_unstale":"2024-09-14T04:26:36.617461+0000","last_undegraded":"2024-09-14T04:26:36.617461+0000","last_fullsized":"2024-09-14T04:26:36.617461+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:10.470781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"427'8213","reported_seq":14470,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.085876+0000","last_change":"2024-09-14T04:26:29.659059+0000","last_active":"2024-09-14T04:26:51.085876+0000","last_peered":"2024-09-14T04:26:51.085876+0000","last_clean":"2024-09-14T04:26:51.085876+0000","last_became_active":"2024-09-14T04:26:29.648121+0000","last_became_peered":"2024-09-14T04:26:29.648121+0000","last_unstale":"2024-09-14T04:26:51.085876+0000","last_undegraded":"2024-09-14T04:26:51.085876+0000","last_fullsized":"2024-09-14T04:26:51.085876+0000","mapping_epoch":425,"log_start":"410'2400","ondisk_log_start":"410'2400","created":398,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:05.353280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432472064,"num_objects":6599,"num_object_clones":0,"num_object_copies":19797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8213,"num_write_kb":473984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":403,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647501+0000","last_change":"2024-09-14T04:25:10.210264+0000","last_active":"2024-09-14T04:26:29.647501+0000","last_peered":"2024-09-14T04:26:29.647501+0000","last_clean":"2024-09-14T04:26:29.647501+0000","last_became_active":"2024-09-14T04:25:10.209992+0000","last_became_peered":"2024-09-14T04:25:10.209992+0000","last_unstale":"2024-09-14T04:26:29.647501+0000","last_undegraded":"2024-09-14T04:26:29.647501+0000","last_fullsized":"2024-09-14T04:26:29.647501+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:50.222369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:50.222369+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":376,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644436+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:26:29.644436+0000","last_peered":"2024-09-14T04:26:29.644436+0000","last_clean":"2024-09-14T04:26:29.644436+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:26:29.644436+0000","last_undegraded":"2024-09-14T04:26:29.644436+0000","last_fullsized":"2024-09-14T04:26:29.644436+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":453,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610959+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:26:36.610959+0000","last_peered":"2024-09-14T04:26:36.610959+0000","last_clean":"2024-09-14T04:26:36.610959+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:26:36.610959+0000","last_undegraded":"2024-09-14T04:26:36.610959+0000","last_fullsized":"2024-09-14T04:26:36.610959+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"427'8203","reported_seq":8240,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:50.135612+0000","last_change":"2024-09-14T04:25:01.992668+0000","last_active":"2024-09-14T04:26:50.135612+0000","last_peered":"2024-09-14T04:26:50.135612+0000","last_clean":"2024-09-14T04:26:50.135612+0000","last_became_active":"2024-09-14T04:25:01.992359+0000","last_became_peered":"2024-09-14T04:25:01.992359+0000","last_unstale":"2024-09-14T04:26:50.135612+0000","last_undegraded":"2024-09-14T04:26:50.135612+0000","last_fullsized":"2024-09-14T04:26:50.135612+0000","mapping_epoch":398,"log_start":"412'3100","ondisk_log_start":"412'3100","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5103,"log_dups_size":0,"ondisk_log_size":5103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:26:08.434277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439418880,"num_objects":6705,"num_object_clones":0,"num_object_copies":20115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8203,"num_write_kb":477056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":741,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608297+0000","last_change":"2024-09-14T04:24:10.193742+0000","last_active":"2024-09-14T04:26:36.608297+0000","last_peered":"2024-09-14T04:26:36.608297+0000","last_clean":"2024-09-14T04:26:36.608297+0000","last_became_active":"2024-09-14T04:24:10.193277+0000","last_became_peered":"2024-09-14T04:24:10.193277+0000","last_unstale":"2024-09-14T04:26:36.608297+0000","last_undegraded":"2024-09-14T04:26:36.608297+0000","last_fullsized":"2024-09-14T04:26:36.608297+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:08.270894+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":457,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610761+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:26:36.610761+0000","last_peered":"2024-09-14T04:26:36.610761+0000","last_clean":"2024-09-14T04:26:36.610761+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:26:36.610761+0000","last_undegraded":"2024-09-14T04:26:36.610761+0000","last_fullsized":"2024-09-14T04:26:36.610761+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":428,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647360+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:26:29.647360+0000","last_peered":"2024-09-14T04:26:29.647360+0000","last_clean":"2024-09-14T04:26:29.647360+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:26:29.647360+0000","last_undegraded":"2024-09-14T04:26:29.647360+0000","last_fullsized":"2024-09-14T04:26:29.647360+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"427'8398","reported_seq":8467,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:52.311690+0000","last_change":"2024-09-14T04:25:01.995435+0000","last_active":"2024-09-14T04:26:52.311690+0000","last_peered":"2024-09-14T04:26:52.311690+0000","last_clean":"2024-09-14T04:26:52.311690+0000","last_became_active":"2024-09-14T04:25:01.995177+0000","last_became_peered":"2024-09-14T04:25:01.995177+0000","last_unstale":"2024-09-14T04:26:52.311690+0000","last_undegraded":"2024-09-14T04:26:52.311690+0000","last_fullsized":"2024-09-14T04:26:52.311690+0000","mapping_epoch":398,"log_start":"406'1800","ondisk_log_start":"406'1800","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:02.315254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431095808,"num_objects":6578,"num_object_clones":0,"num_object_copies":19734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8398,"num_write_kb":479232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":661,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643688+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:26:29.643688+0000","last_peered":"2024-09-14T04:26:29.643688+0000","last_clean":"2024-09-14T04:26:29.643688+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:26:29.643688+0000","last_undegraded":"2024-09-14T04:26:29.643688+0000","last_fullsized":"2024-09-14T04:26:29.643688+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":535,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.604157+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:26:36.604157+0000","last_peered":"2024-09-14T04:26:36.604157+0000","last_clean":"2024-09-14T04:26:36.604157+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:26:36.604157+0000","last_undegraded":"2024-09-14T04:26:36.604157+0000","last_fullsized":"2024-09-14T04:26:36.604157+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647867+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:26:29.647867+0000","last_peered":"2024-09-14T04:26:29.647867+0000","last_clean":"2024-09-14T04:26:29.647867+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:26:29.647867+0000","last_undegraded":"2024-09-14T04:26:29.647867+0000","last_fullsized":"2024-09-14T04:26:29.647867+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"427'8218","reported_seq":12833,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:50.103694+0000","last_change":"2024-09-14T04:25:44.643768+0000","last_active":"2024-09-14T04:26:50.103694+0000","last_peered":"2024-09-14T04:26:50.103694+0000","last_clean":"2024-09-14T04:26:50.103694+0000","last_became_active":"2024-09-14T04:25:44.638986+0000","last_became_peered":"2024-09-14T04:25:44.638986+0000","last_unstale":"2024-09-14T04:26:50.103694+0000","last_undegraded":"2024-09-14T04:26:50.103694+0000","last_fullsized":"2024-09-14T04:26:50.103694+0000","mapping_epoch":419,"log_start":"412'3100","ondisk_log_start":"412'3100","created":398,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:31:07.177695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437256192,"num_objects":6672,"num_object_clones":0,"num_object_copies":20016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8218,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":927,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613842+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:26:36.613842+0000","last_peered":"2024-09-14T04:26:36.613842+0000","last_clean":"2024-09-14T04:26:36.613842+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:26:36.613842+0000","last_undegraded":"2024-09-14T04:26:36.613842+0000","last_fullsized":"2024-09-14T04:26:36.613842+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647166+0000","last_change":"2024-09-14T04:25:07.967648+0000","last_active":"2024-09-14T04:26:29.647166+0000","last_peered":"2024-09-14T04:26:29.647166+0000","last_clean":"2024-09-14T04:26:29.647166+0000","last_became_active":"2024-09-14T04:25:07.967284+0000","last_became_peered":"2024-09-14T04:25:07.967284+0000","last_unstale":"2024-09-14T04:26:29.647166+0000","last_undegraded":"2024-09-14T04:26:29.647166+0000","last_fullsized":"2024-09-14T04:26:29.647166+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:29.967190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":434,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647637+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:26:29.647637+0000","last_peered":"2024-09-14T04:26:29.647637+0000","last_clean":"2024-09-14T04:26:29.647637+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:26:29.647637+0000","last_undegraded":"2024-09-14T04:26:29.647637+0000","last_fullsized":"2024-09-14T04:26:29.647637+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"427'8123","reported_seq":8161,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:50.100321+0000","last_change":"2024-09-14T04:25:02.146105+0000","last_active":"2024-09-14T04:26:50.100321+0000","last_peered":"2024-09-14T04:26:50.100321+0000","last_clean":"2024-09-14T04:26:50.100321+0000","last_became_active":"2024-09-14T04:25:02.145736+0000","last_became_peered":"2024-09-14T04:25:02.145736+0000","last_unstale":"2024-09-14T04:26:50.100321+0000","last_undegraded":"2024-09-14T04:26:50.100321+0000","last_fullsized":"2024-09-14T04:26:50.100321+0000","mapping_epoch":398,"log_start":"412'3000","ondisk_log_start":"412'3000","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5123,"log_dups_size":0,"ondisk_log_size":5123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:48:35.907164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436797440,"num_objects":6665,"num_object_clones":0,"num_object_copies":19995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8123,"num_write_kb":473216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":632,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644039+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:26:29.644039+0000","last_peered":"2024-09-14T04:26:29.644039+0000","last_clean":"2024-09-14T04:26:29.644039+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:26:29.644039+0000","last_undegraded":"2024-09-14T04:26:29.644039+0000","last_fullsized":"2024-09-14T04:26:29.644039+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609291+0000","last_change":"2024-09-14T04:25:35.561629+0000","last_active":"2024-09-14T04:26:36.609291+0000","last_peered":"2024-09-14T04:26:36.609291+0000","last_clean":"2024-09-14T04:26:36.609291+0000","last_became_active":"2024-09-14T04:25:35.561305+0000","last_became_peered":"2024-09-14T04:25:35.561305+0000","last_unstale":"2024-09-14T04:26:36.609291+0000","last_undegraded":"2024-09-14T04:26:36.609291+0000","last_fullsized":"2024-09-14T04:26:36.609291+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:16:39.574054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":696,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614968+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:26:36.614968+0000","last_peered":"2024-09-14T04:26:36.614968+0000","last_clean":"2024-09-14T04:26:36.614968+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:26:36.614968+0000","last_undegraded":"2024-09-14T04:26:36.614968+0000","last_fullsized":"2024-09-14T04:26:36.614968+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"427'8335","reported_seq":8404,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.077861+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:26:51.077861+0000","last_peered":"2024-09-14T04:26:51.077861+0000","last_clean":"2024-09-14T04:26:51.077861+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:26:51.077861+0000","last_undegraded":"2024-09-14T04:26:51.077861+0000","last_fullsized":"2024-09-14T04:26:51.077861+0000","mapping_epoch":398,"log_start":"412'3100","ondisk_log_start":"412'3100","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5235,"log_dups_size":0,"ondisk_log_size":5235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:09.181654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439943168,"num_objects":6713,"num_object_clones":0,"num_object_copies":20139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8335,"num_write_kb":481536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":796,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610250+0000","last_change":"2024-09-14T04:22:16.586702+0000","last_active":"2024-09-14T04:26:36.610250+0000","last_peered":"2024-09-14T04:26:36.610250+0000","last_clean":"2024-09-14T04:26:36.610250+0000","last_became_active":"2024-09-14T04:22:14.380322+0000","last_became_peered":"2024-09-14T04:22:14.380322+0000","last_unstale":"2024-09-14T04:26:36.610250+0000","last_undegraded":"2024-09-14T04:26:36.610250+0000","last_fullsized":"2024-09-14T04:26:36.610250+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:17.530093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":366,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644179+0000","last_change":"2024-09-14T04:24:08.979706+0000","last_active":"2024-09-14T04:26:29.644179+0000","last_peered":"2024-09-14T04:26:29.644179+0000","last_clean":"2024-09-14T04:26:29.644179+0000","last_became_active":"2024-09-14T04:24:08.979397+0000","last_became_peered":"2024-09-14T04:24:08.979397+0000","last_unstale":"2024-09-14T04:26:29.644179+0000","last_undegraded":"2024-09-14T04:26:29.644179+0000","last_fullsized":"2024-09-14T04:26:29.644179+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:12.232467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609238+0000","last_change":"2024-09-14T04:25:52.594667+0000","last_active":"2024-09-14T04:26:36.609238+0000","last_peered":"2024-09-14T04:26:36.609238+0000","last_clean":"2024-09-14T04:26:36.609238+0000","last_became_active":"2024-09-14T04:25:52.594400+0000","last_became_peered":"2024-09-14T04:25:52.594400+0000","last_unstale":"2024-09-14T04:26:36.609238+0000","last_undegraded":"2024-09-14T04:26:36.609238+0000","last_fullsized":"2024-09-14T04:26:36.609238+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:12.710417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647785+0000","last_change":"2024-09-14T04:25:35.560355+0000","last_active":"2024-09-14T04:26:29.647785+0000","last_peered":"2024-09-14T04:26:29.647785+0000","last_clean":"2024-09-14T04:26:29.647785+0000","last_became_active":"2024-09-14T04:25:35.559975+0000","last_became_peered":"2024-09-14T04:25:35.559975+0000","last_unstale":"2024-09-14T04:26:29.647785+0000","last_undegraded":"2024-09-14T04:26:29.647785+0000","last_fullsized":"2024-09-14T04:26:29.647785+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:58.033714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"427'8233","reported_seq":8308,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:52.333260+0000","last_change":"2024-09-14T04:25:07.966414+0000","last_active":"2024-09-14T04:26:52.333260+0000","last_peered":"2024-09-14T04:26:52.333260+0000","last_clean":"2024-09-14T04:26:52.333260+0000","last_became_active":"2024-09-14T04:25:07.966014+0000","last_became_peered":"2024-09-14T04:25:07.966014+0000","last_unstale":"2024-09-14T04:26:52.333260+0000","last_undegraded":"2024-09-14T04:26:52.333260+0000","last_fullsized":"2024-09-14T04:26:52.333260+0000","mapping_epoch":401,"log_start":"406'1700","ondisk_log_start":"406'1700","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:58.062778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423952384,"num_objects":6469,"num_object_clones":0,"num_object_copies":19407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8233,"num_write_kb":470464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":918,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613940+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:26:36.613940+0000","last_peered":"2024-09-14T04:26:36.613940+0000","last_clean":"2024-09-14T04:26:36.613940+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:26:36.613940+0000","last_undegraded":"2024-09-14T04:26:36.613940+0000","last_fullsized":"2024-09-14T04:26:36.613940+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":397,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613255+0000","last_change":"2024-09-14T04:22:11.332840+0000","last_active":"2024-09-14T04:26:36.613255+0000","last_peered":"2024-09-14T04:26:36.613255+0000","last_clean":"2024-09-14T04:26:36.613255+0000","last_became_active":"2024-09-14T04:22:11.332416+0000","last_became_peered":"2024-09-14T04:22:11.332416+0000","last_unstale":"2024-09-14T04:26:36.613255+0000","last_undegraded":"2024-09-14T04:26:36.613255+0000","last_fullsized":"2024-09-14T04:26:36.613255+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:48.504372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":702,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.606712+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:26:36.606712+0000","last_peered":"2024-09-14T04:26:36.606712+0000","last_clean":"2024-09-14T04:26:36.606712+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:26:36.606712+0000","last_undegraded":"2024-09-14T04:26:36.606712+0000","last_fullsized":"2024-09-14T04:26:36.606712+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":464,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612605+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:26:36.612605+0000","last_peered":"2024-09-14T04:26:36.612605+0000","last_clean":"2024-09-14T04:26:36.612605+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:26:36.612605+0000","last_undegraded":"2024-09-14T04:26:36.612605+0000","last_fullsized":"2024-09-14T04:26:36.612605+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"427'8098","reported_seq":8136,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.114210+0000","last_change":"2024-09-14T04:25:01.992854+0000","last_active":"2024-09-14T04:26:51.114210+0000","last_peered":"2024-09-14T04:26:51.114210+0000","last_clean":"2024-09-14T04:26:51.114210+0000","last_became_active":"2024-09-14T04:25:01.992514+0000","last_became_peered":"2024-09-14T04:25:01.992514+0000","last_unstale":"2024-09-14T04:26:51.114210+0000","last_undegraded":"2024-09-14T04:26:51.114210+0000","last_fullsized":"2024-09-14T04:26:51.114210+0000","mapping_epoch":398,"log_start":"410'2300","ondisk_log_start":"410'2300","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:59.789126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425721856,"num_objects":6496,"num_object_clones":0,"num_object_copies":19488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8098,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":671,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.604014+0000","last_change":"2024-09-14T04:25:10.213700+0000","last_active":"2024-09-14T04:26:36.604014+0000","last_peered":"2024-09-14T04:26:36.604014+0000","last_clean":"2024-09-14T04:26:36.604014+0000","last_became_active":"2024-09-14T04:25:10.213291+0000","last_became_peered":"2024-09-14T04:25:10.213291+0000","last_unstale":"2024-09-14T04:26:36.604014+0000","last_undegraded":"2024-09-14T04:26:36.604014+0000","last_fullsized":"2024-09-14T04:26:36.604014+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:58.394641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":670,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608438+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:26:36.608438+0000","last_peered":"2024-09-14T04:26:36.608438+0000","last_clean":"2024-09-14T04:26:36.608438+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:26:36.608438+0000","last_undegraded":"2024-09-14T04:26:36.608438+0000","last_fullsized":"2024-09-14T04:26:36.608438+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":701,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613398+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:26:36.613398+0000","last_peered":"2024-09-14T04:26:36.613398+0000","last_clean":"2024-09-14T04:26:36.613398+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:26:36.613398+0000","last_undegraded":"2024-09-14T04:26:36.613398+0000","last_fullsized":"2024-09-14T04:26:36.613398+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614777+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:26:36.614777+0000","last_peered":"2024-09-14T04:26:36.614777+0000","last_clean":"2024-09-14T04:26:36.614777+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:26:36.614777+0000","last_undegraded":"2024-09-14T04:26:36.614777+0000","last_fullsized":"2024-09-14T04:26:36.614777+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"427'8211","reported_seq":8250,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:50.117924+0000","last_change":"2024-09-14T04:25:02.146113+0000","last_active":"2024-09-14T04:26:50.117924+0000","last_peered":"2024-09-14T04:26:50.117924+0000","last_clean":"2024-09-14T04:26:50.117924+0000","last_became_active":"2024-09-14T04:25:02.145762+0000","last_became_peered":"2024-09-14T04:25:02.145762+0000","last_unstale":"2024-09-14T04:26:50.117924+0000","last_undegraded":"2024-09-14T04:26:50.117924+0000","last_fullsized":"2024-09-14T04:26:50.117924+0000","mapping_epoch":398,"log_start":"415'3100","ondisk_log_start":"415'3100","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:33.000554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441057304,"num_objects":6731,"num_object_clones":0,"num_object_copies":20193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6731,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8211,"num_write_kb":478081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":655,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643642+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:26:29.643642+0000","last_peered":"2024-09-14T04:26:29.643642+0000","last_clean":"2024-09-14T04:26:29.643642+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:26:29.643642+0000","last_undegraded":"2024-09-14T04:26:29.643642+0000","last_fullsized":"2024-09-14T04:26:29.643642+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":363,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643889+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:26:29.643889+0000","last_peered":"2024-09-14T04:26:29.643889+0000","last_clean":"2024-09-14T04:26:29.643889+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:26:29.643889+0000","last_undegraded":"2024-09-14T04:26:29.643889+0000","last_fullsized":"2024-09-14T04:26:29.643889+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":446,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647506+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:26:29.647506+0000","last_peered":"2024-09-14T04:26:29.647506+0000","last_clean":"2024-09-14T04:26:29.647506+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:26:29.647506+0000","last_undegraded":"2024-09-14T04:26:29.647506+0000","last_fullsized":"2024-09-14T04:26:29.647506+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":383,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644535+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:26:29.644535+0000","last_peered":"2024-09-14T04:26:29.644535+0000","last_clean":"2024-09-14T04:26:29.644535+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:26:29.644535+0000","last_undegraded":"2024-09-14T04:26:29.644535+0000","last_fullsized":"2024-09-14T04:26:29.644535+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"427'8338","reported_seq":8374,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.119846+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:26:51.119846+0000","last_peered":"2024-09-14T04:26:51.119846+0000","last_clean":"2024-09-14T04:26:51.119846+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:26:51.119846+0000","last_undegraded":"2024-09-14T04:26:51.119846+0000","last_fullsized":"2024-09-14T04:26:51.119846+0000","mapping_epoch":398,"log_start":"411'2500","ondisk_log_start":"411'2500","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435027968,"num_objects":6638,"num_object_clones":0,"num_object_copies":19914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8338,"num_write_kb":479232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":690,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617990+0000","last_change":"2024-09-14T04:25:38.633170+0000","last_active":"2024-09-14T04:26:36.617990+0000","last_peered":"2024-09-14T04:26:36.617990+0000","last_clean":"2024-09-14T04:26:36.617990+0000","last_became_active":"2024-09-14T04:25:38.632816+0000","last_became_peered":"2024-09-14T04:25:38.632816+0000","last_unstale":"2024-09-14T04:26:36.617990+0000","last_undegraded":"2024-09-14T04:26:36.617990+0000","last_fullsized":"2024-09-14T04:26:36.617990+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:55:39.218376+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":392,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644959+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:26:29.644959+0000","last_peered":"2024-09-14T04:26:29.644959+0000","last_clean":"2024-09-14T04:26:29.644959+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:26:29.644959+0000","last_undegraded":"2024-09-14T04:26:29.644959+0000","last_fullsized":"2024-09-14T04:26:29.644959+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":758,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618982+0000","last_change":"2024-09-14T04:22:11.330430+0000","last_active":"2024-09-14T04:26:36.618982+0000","last_peered":"2024-09-14T04:26:36.618982+0000","last_clean":"2024-09-14T04:26:36.618982+0000","last_became_active":"2024-09-14T04:22:11.330093+0000","last_became_peered":"2024-09-14T04:22:11.330093+0000","last_unstale":"2024-09-14T04:26:36.618982+0000","last_undegraded":"2024-09-14T04:26:36.618982+0000","last_fullsized":"2024-09-14T04:26:36.618982+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:39.386753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617324+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:26:36.617324+0000","last_peered":"2024-09-14T04:26:36.617324+0000","last_clean":"2024-09-14T04:26:36.617324+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:26:36.617324+0000","last_undegraded":"2024-09-14T04:26:36.617324+0000","last_fullsized":"2024-09-14T04:26:36.617324+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"427'8176","reported_seq":8244,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:52.327672+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:26:52.327672+0000","last_peered":"2024-09-14T04:26:52.327672+0000","last_clean":"2024-09-14T04:26:52.327672+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:26:52.327672+0000","last_undegraded":"2024-09-14T04:26:52.327672+0000","last_fullsized":"2024-09-14T04:26:52.327672+0000","mapping_epoch":398,"log_start":"406'1600","ondisk_log_start":"406'1600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6576,"log_dups_size":0,"ondisk_log_size":6576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:13.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421789696,"num_objects":6436,"num_object_clones":0,"num_object_copies":19308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8176,"num_write_kb":467584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":708,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612174+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:26:36.612174+0000","last_peered":"2024-09-14T04:26:36.612174+0000","last_clean":"2024-09-14T04:26:36.612174+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:26:36.612174+0000","last_undegraded":"2024-09-14T04:26:36.612174+0000","last_fullsized":"2024-09-14T04:26:36.612174+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":704,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618962+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:26:36.618962+0000","last_peered":"2024-09-14T04:26:36.618962+0000","last_clean":"2024-09-14T04:26:36.618962+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:26:36.618962+0000","last_undegraded":"2024-09-14T04:26:36.618962+0000","last_fullsized":"2024-09-14T04:26:36.618962+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":436,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.618630+0000","last_change":"2024-09-14T04:22:11.331756+0000","last_active":"2024-09-14T04:26:36.618630+0000","last_peered":"2024-09-14T04:26:36.618630+0000","last_clean":"2024-09-14T04:26:36.618630+0000","last_became_active":"2024-09-14T04:22:11.331415+0000","last_became_peered":"2024-09-14T04:22:11.331415+0000","last_unstale":"2024-09-14T04:26:36.618630+0000","last_undegraded":"2024-09-14T04:26:36.618630+0000","last_fullsized":"2024-09-14T04:26:36.618630+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:56:07.929419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":469,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610874+0000","last_change":"2024-09-14T04:24:08.989110+0000","last_active":"2024-09-14T04:26:36.610874+0000","last_peered":"2024-09-14T04:26:36.610874+0000","last_clean":"2024-09-14T04:26:36.610874+0000","last_became_active":"2024-09-14T04:24:08.987440+0000","last_became_peered":"2024-09-14T04:24:08.987440+0000","last_unstale":"2024-09-14T04:26:36.610874+0000","last_undegraded":"2024-09-14T04:26:36.610874+0000","last_fullsized":"2024-09-14T04:26:36.610874+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:17:32.812292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"427'8105","reported_seq":8143,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:51.105908+0000","last_change":"2024-09-14T04:25:01.993057+0000","last_active":"2024-09-14T04:26:51.105908+0000","last_peered":"2024-09-14T04:26:51.105908+0000","last_clean":"2024-09-14T04:26:51.105908+0000","last_became_active":"2024-09-14T04:25:01.992668+0000","last_became_peered":"2024-09-14T04:25:01.992668+0000","last_unstale":"2024-09-14T04:26:51.105908+0000","last_undegraded":"2024-09-14T04:26:51.105908+0000","last_fullsized":"2024-09-14T04:26:51.105908+0000","mapping_epoch":398,"log_start":"410'2300","ondisk_log_start":"410'2300","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:38.544577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428408832,"num_objects":6537,"num_object_clones":0,"num_object_copies":19611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8105,"num_write_kb":468544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":732,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647627+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:26:29.647627+0000","last_peered":"2024-09-14T04:26:29.647627+0000","last_clean":"2024-09-14T04:26:29.647627+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:26:29.647627+0000","last_undegraded":"2024-09-14T04:26:29.647627+0000","last_fullsized":"2024-09-14T04:26:29.647627+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":359,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.611710+0000","last_change":"2024-09-14T04:24:08.999455+0000","last_active":"2024-09-14T04:26:36.611710+0000","last_peered":"2024-09-14T04:26:36.611710+0000","last_clean":"2024-09-14T04:26:36.611710+0000","last_became_active":"2024-09-14T04:24:08.994093+0000","last_became_peered":"2024-09-14T04:24:08.994093+0000","last_unstale":"2024-09-14T04:26:36.611710+0000","last_undegraded":"2024-09-14T04:26:36.611710+0000","last_fullsized":"2024-09-14T04:26:36.611710+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:29:31.948482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":690,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.614508+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:26:36.614508+0000","last_peered":"2024-09-14T04:26:36.614508+0000","last_clean":"2024-09-14T04:26:36.614508+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:26:36.614508+0000","last_undegraded":"2024-09-14T04:26:36.614508+0000","last_fullsized":"2024-09-14T04:26:36.614508+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":389,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609977+0000","last_change":"2024-09-14T04:25:35.562908+0000","last_active":"2024-09-14T04:26:36.609977+0000","last_peered":"2024-09-14T04:26:36.609977+0000","last_clean":"2024-09-14T04:26:36.609977+0000","last_became_active":"2024-09-14T04:25:35.562023+0000","last_became_peered":"2024-09-14T04:25:35.562023+0000","last_unstale":"2024-09-14T04:26:36.609977+0000","last_undegraded":"2024-09-14T04:26:36.609977+0000","last_fullsized":"2024-09-14T04:26:36.609977+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:55.656734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.8","version":"427'8161","reported_seq":8230,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:52.317686+0000","last_change":"2024-09-14T04:25:01.996105+0000","last_active":"2024-09-14T04:26:52.317686+0000","last_peered":"2024-09-14T04:26:52.317686+0000","last_clean":"2024-09-14T04:26:52.317686+0000","last_became_active":"2024-09-14T04:25:01.995811+0000","last_became_peered":"2024-09-14T04:25:01.995811+0000","last_unstale":"2024-09-14T04:26:52.317686+0000","last_undegraded":"2024-09-14T04:26:52.317686+0000","last_fullsized":"2024-09-14T04:26:52.317686+0000","mapping_epoch":398,"log_start":"406'1600","ondisk_log_start":"406'1600","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:09:04.759721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418316288,"num_objects":6383,"num_object_clones":0,"num_object_copies":19149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8161,"num_write_kb":465408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":979,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617841+0000","last_change":"2024-09-14T04:25:28.605221+0000","last_active":"2024-09-14T04:26:36.617841+0000","last_peered":"2024-09-14T04:26:36.617841+0000","last_clean":"2024-09-14T04:26:36.617841+0000","last_became_active":"2024-09-14T04:25:28.604819+0000","last_became_peered":"2024-09-14T04:25:28.604819+0000","last_unstale":"2024-09-14T04:26:36.617841+0000","last_undegraded":"2024-09-14T04:26:36.617841+0000","last_fullsized":"2024-09-14T04:26:36.617841+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:21.033138+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":468,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612433+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:26:36.612433+0000","last_peered":"2024-09-14T04:26:36.612433+0000","last_clean":"2024-09-14T04:26:36.612433+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:26:36.612433+0000","last_undegraded":"2024-09-14T04:26:36.612433+0000","last_fullsized":"2024-09-14T04:26:36.612433+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":535,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608226+0000","last_change":"2024-09-14T04:23:27.596032+0000","last_active":"2024-09-14T04:26:36.608226+0000","last_peered":"2024-09-14T04:26:36.608226+0000","last_clean":"2024-09-14T04:26:36.608226+0000","last_became_active":"2024-09-14T04:23:27.595521+0000","last_became_peered":"2024-09-14T04:23:27.595521+0000","last_unstale":"2024-09-14T04:26:36.608226+0000","last_undegraded":"2024-09-14T04:26:36.608226+0000","last_fullsized":"2024-09-14T04:26:36.608226+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:14.145527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":412,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646961+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:26:29.646961+0000","last_peered":"2024-09-14T04:26:29.646961+0000","last_clean":"2024-09-14T04:26:29.646961+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:26:29.646961+0000","last_undegraded":"2024-09-14T04:26:29.646961+0000","last_fullsized":"2024-09-14T04:26:29.646961+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":448,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647319+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:26:29.647319+0000","last_peered":"2024-09-14T04:26:29.647319+0000","last_clean":"2024-09-14T04:26:29.647319+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:26:29.647319+0000","last_undegraded":"2024-09-14T04:26:29.647319+0000","last_fullsized":"2024-09-14T04:26:29.647319+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":447,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647444+0000","last_change":"2024-09-14T04:22:11.342624+0000","last_active":"2024-09-14T04:26:29.647444+0000","last_peered":"2024-09-14T04:26:29.647444+0000","last_clean":"2024-09-14T04:26:29.647444+0000","last_became_active":"2024-09-14T04:22:11.329897+0000","last_became_peered":"2024-09-14T04:22:11.329897+0000","last_unstale":"2024-09-14T04:26:29.647444+0000","last_undegraded":"2024-09-14T04:26:29.647444+0000","last_fullsized":"2024-09-14T04:26:29.647444+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:28.744133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.617123+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:26:36.617123+0000","last_peered":"2024-09-14T04:26:36.617123+0000","last_clean":"2024-09-14T04:26:36.617123+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:26:36.617123+0000","last_undegraded":"2024-09-14T04:26:36.617123+0000","last_fullsized":"2024-09-14T04:26:36.617123+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":373,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.625385+0000","last_change":"2024-09-14T04:22:11.345841+0000","last_active":"2024-09-14T04:26:36.625385+0000","last_peered":"2024-09-14T04:26:36.625385+0000","last_clean":"2024-09-14T04:26:36.625385+0000","last_became_active":"2024-09-14T04:22:11.336227+0000","last_became_peered":"2024-09-14T04:22:11.336227+0000","last_unstale":"2024-09-14T04:26:36.625385+0000","last_undegraded":"2024-09-14T04:26:36.625385+0000","last_fullsized":"2024-09-14T04:26:36.625385+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:13.358517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":914,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613771+0000","last_change":"2024-09-14T04:25:10.214026+0000","last_active":"2024-09-14T04:26:36.613771+0000","last_peered":"2024-09-14T04:26:36.613771+0000","last_clean":"2024-09-14T04:26:36.613771+0000","last_became_active":"2024-09-14T04:25:10.212511+0000","last_became_peered":"2024-09-14T04:25:10.212511+0000","last_unstale":"2024-09-14T04:26:36.613771+0000","last_undegraded":"2024-09-14T04:26:36.613771+0000","last_fullsized":"2024-09-14T04:26:36.613771+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:20.633170+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":319,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:31.086198+0000","last_change":"2024-09-14T04:25:35.559702+0000","last_active":"2024-09-14T04:26:31.086198+0000","last_peered":"2024-09-14T04:26:31.086198+0000","last_clean":"2024-09-14T04:26:31.086198+0000","last_became_active":"2024-09-14T04:25:35.559395+0000","last_became_peered":"2024-09-14T04:25:35.559395+0000","last_unstale":"2024-09-14T04:26:31.086198+0000","last_undegraded":"2024-09-14T04:26:31.086198+0000","last_fullsized":"2024-09-14T04:26:31.086198+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:51.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":309,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:31.086033+0000","last_change":"2024-09-14T04:24:09.122059+0000","last_active":"2024-09-14T04:26:31.086033+0000","last_peered":"2024-09-14T04:26:31.086033+0000","last_clean":"2024-09-14T04:26:31.086033+0000","last_became_active":"2024-09-14T04:24:09.121824+0000","last_became_peered":"2024-09-14T04:24:09.121824+0000","last_unstale":"2024-09-14T04:26:31.086033+0000","last_undegraded":"2024-09-14T04:26:31.086033+0000","last_fullsized":"2024-09-14T04:26:31.086033+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:36.179815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":463,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612012+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:26:36.612012+0000","last_peered":"2024-09-14T04:26:36.612012+0000","last_clean":"2024-09-14T04:26:36.612012+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:26:36.612012+0000","last_undegraded":"2024-09-14T04:26:36.612012+0000","last_fullsized":"2024-09-14T04:26:36.612012+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":567,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.609629+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:26:36.609629+0000","last_peered":"2024-09-14T04:26:36.609629+0000","last_clean":"2024-09-14T04:26:36.609629+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:26:36.609629+0000","last_undegraded":"2024-09-14T04:26:36.609629+0000","last_fullsized":"2024-09-14T04:26:36.609629+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.644832+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:26:29.644832+0000","last_peered":"2024-09-14T04:26:29.644832+0000","last_clean":"2024-09-14T04:26:29.644832+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:26:29.644832+0000","last_undegraded":"2024-09-14T04:26:29.644832+0000","last_fullsized":"2024-09-14T04:26:29.644832+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.643515+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:26:29.643515+0000","last_peered":"2024-09-14T04:26:29.643515+0000","last_clean":"2024-09-14T04:26:29.643515+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:26:29.643515+0000","last_undegraded":"2024-09-14T04:26:29.643515+0000","last_fullsized":"2024-09-14T04:26:29.643515+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":480,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612384+0000","last_change":"2024-09-14T04:25:35.564324+0000","last_active":"2024-09-14T04:26:36.612384+0000","last_peered":"2024-09-14T04:26:36.612384+0000","last_clean":"2024-09-14T04:26:36.612384+0000","last_became_active":"2024-09-14T04:25:35.562918+0000","last_became_peered":"2024-09-14T04:25:35.562918+0000","last_unstale":"2024-09-14T04:26:36.612384+0000","last_undegraded":"2024-09-14T04:26:36.612384+0000","last_fullsized":"2024-09-14T04:26:36.612384+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:16.172185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":727,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613575+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:26:36.613575+0000","last_peered":"2024-09-14T04:26:36.613575+0000","last_clean":"2024-09-14T04:26:36.613575+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:26:36.613575+0000","last_undegraded":"2024-09-14T04:26:36.613575+0000","last_fullsized":"2024-09-14T04:26:36.613575+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":277,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091862+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:26:36.091862+0000","last_peered":"2024-09-14T04:26:36.091862+0000","last_clean":"2024-09-14T04:26:36.091862+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:26:36.091862+0000","last_undegraded":"2024-09-14T04:26:36.091862+0000","last_fullsized":"2024-09-14T04:26:36.091862+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646729+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:26:29.646729+0000","last_peered":"2024-09-14T04:26:29.646729+0000","last_clean":"2024-09-14T04:26:29.646729+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:26:29.646729+0000","last_undegraded":"2024-09-14T04:26:29.646729+0000","last_fullsized":"2024-09-14T04:26:29.646729+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":139,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.613901+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:26:36.613901+0000","last_peered":"2024-09-14T04:26:36.613901+0000","last_clean":"2024-09-14T04:26:36.613901+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:26:36.613901+0000","last_undegraded":"2024-09-14T04:26:36.613901+0000","last_fullsized":"2024-09-14T04:26:36.613901+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":477,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.610814+0000","last_change":"2024-09-14T04:23:27.573928+0000","last_active":"2024-09-14T04:26:36.610814+0000","last_peered":"2024-09-14T04:26:36.610814+0000","last_clean":"2024-09-14T04:26:36.610814+0000","last_became_active":"2024-09-14T04:23:27.573557+0000","last_became_peered":"2024-09-14T04:23:27.573557+0000","last_unstale":"2024-09-14T04:26:36.610814+0000","last_undegraded":"2024-09-14T04:26:36.610814+0000","last_fullsized":"2024-09-14T04:26:36.610814+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:07.509590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646983+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:26:29.646983+0000","last_peered":"2024-09-14T04:26:29.646983+0000","last_clean":"2024-09-14T04:26:29.646983+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:26:29.646983+0000","last_undegraded":"2024-09-14T04:26:29.646983+0000","last_fullsized":"2024-09-14T04:26:29.646983+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":375,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091762+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:26:36.091762+0000","last_peered":"2024-09-14T04:26:36.091762+0000","last_clean":"2024-09-14T04:26:36.091762+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:26:36.091762+0000","last_undegraded":"2024-09-14T04:26:36.091762+0000","last_fullsized":"2024-09-14T04:26:36.091762+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.647027+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:26:29.647027+0000","last_peered":"2024-09-14T04:26:29.647027+0000","last_clean":"2024-09-14T04:26:29.647027+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:26:29.647027+0000","last_undegraded":"2024-09-14T04:26:29.647027+0000","last_fullsized":"2024-09-14T04:26:29.647027+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":384,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.091841+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:26:36.091841+0000","last_peered":"2024-09-14T04:26:36.091841+0000","last_clean":"2024-09-14T04:26:36.091841+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:26:36.091841+0000","last_undegraded":"2024-09-14T04:26:36.091841+0000","last_fullsized":"2024-09-14T04:26:36.091841+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":616,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608350+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:26:36.608350+0000","last_peered":"2024-09-14T04:26:36.608350+0000","last_clean":"2024-09-14T04:26:36.608350+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:26:36.608350+0000","last_undegraded":"2024-09-14T04:26:36.608350+0000","last_fullsized":"2024-09-14T04:26:36.608350+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":288,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.608377+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:26:36.608377+0000","last_peered":"2024-09-14T04:26:36.608377+0000","last_clean":"2024-09-14T04:26:36.608377+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:26:36.608377+0000","last_undegraded":"2024-09-14T04:26:36.608377+0000","last_fullsized":"2024-09-14T04:26:36.608377+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":409,"reported_epoch":426,"state":"active+clean","last_fresh":"2024-09-14T04:26:29.646709+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:26:29.646709+0000","last_peered":"2024-09-14T04:26:29.646709+0000","last_clean":"2024-09-14T04:26:29.646709+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:26:29.646709+0000","last_undegraded":"2024-09-14T04:26:29.646709+0000","last_fullsized":"2024-09-14T04:26:29.646709+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":511,"reported_epoch":427,"state":"active+clean","last_fresh":"2024-09-14T04:26:36.612098+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:26:36.612098+0000","last_peered":"2024-09-14T04:26:36.612098+0000","last_clean":"2024-09-14T04:26:36.612098+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:26:36.612098+0000","last_undegraded":"2024-09-14T04:26:36.612098+0000","last_fullsized":"2024-09-14T04:26:36.612098+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":6896746520,"num_objects":105237,"num_object_clones":0,"num_object_copies":315711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105237,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131247,"num_write_kb":7567425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5374,"num_bytes_recovered":352190464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20897411072,"data_stored":20897398856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92547,"ondisk_log_size":92547,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":12322,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2506969,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201835,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463473,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834146,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4919480,"kb_used_data":4689456,"kb_used_omap":5,"kb_used_meta":230010,"kb_avail":88825672,"statfs":{"total":95995035648,"available":90957488128,"internally_reserved":0,"allocated":4802002944,"data_stored":4799241564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":235530301},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986745,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5627784,"kb_used_data":4265344,"kb_used_omap":1,"kb_used_meta":1362430,"kb_avail":88117368,"statfs":{"total":95995035648,"available":90232184832,"internally_reserved":0,"allocated":4367712256,"data_stored":4364938281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1395128774},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748774,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3533116,"kb_used_data":2547316,"kb_used_omap":4,"kb_used_meta":985787,"kb_avail":90212036,"statfs":{"total":95995035648,"available":92377124864,"internally_reserved":0,"allocated":2608451584,"data_stored":2605685914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1009446444},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605286995,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4272068,"kb_used_data":3751100,"kb_used_omap":10,"kb_used_meta":520629,"kb_avail":89473084,"statfs":{"total":95995035648,"available":91620438016,"internally_reserved":0,"allocated":3841126400,"data_stored":3838357726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":533124695},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468317,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3596084,"kb_used_data":2564588,"kb_used_omap":5,"kb_used_meta":1031482,"kb_avail":90149068,"statfs":{"total":95995035648,"available":92312645632,"internally_reserved":0,"allocated":2626138112,"data_stored":2623378551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1056237919},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,2,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342432,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3425228,"kb_used_data":2767812,"kb_used_omap":10,"kb_used_meta":657397,"kb_avail":90319924,"statfs":{"total":95995035648,"available":92487602176,"internally_reserved":0,"allocated":2834239488,"data_stored":2831462209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":673174738},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2796224512,"data_stored":2796224512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2595033088,"data_stored":2595029016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3816685568,"data_stored":3816685568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2587037696,"data_stored":2587033624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4350414848,"data_stored":4350410776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4752015360,"data_stored":4752015360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:26:58.601 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:26:58.601 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_8 pg_num 26 2024-09-14T04:26:58.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:58.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:58.738 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:58.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:26:58.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:26:58 smithi012 ceph-mon[101180]: pgmap v1430: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:58.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:26:58.889 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:58.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:26:58.924 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:26:58 smithi001 ceph-mon[107399]: pgmap v1430: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:58.924 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:26:58 smithi001 ceph-mon[112165]: pgmap v1430: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:26:58.983 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:26:58.991 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:59.013 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:26:59.013 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:26:59.070 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:59.172 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:59.215 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:59.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:26:59.375 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:26:59.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:26:59.509 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:26:59.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:26:59.652 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:26:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:26:59.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:26:59.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:26:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:59.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:26:59.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:26:59.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:26:59.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:26:59.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:00.019 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:00.033 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:00.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:00.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:27:00.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:00.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:00.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:00.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:00.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:00.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:00.476 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:00.479 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:00.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:00.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:00.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:00.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:00 smithi001 ceph-mon[107399]: pgmap v1431: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:27:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:00 smithi001 ceph-mon[112165]: pgmap v1431: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:27:00.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:00.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:00 smithi012 ceph-mon[101180]: pgmap v1431: 109 pgs: 109 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-14T04:27:00.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:00.870 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:00.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:00.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:01.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:01.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.159 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:01.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:01.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:01.297 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:01.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:01.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:01.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:01.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:01.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:01.635 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:01.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:01.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:01.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:01.838 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:01.940 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:01.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:01.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:02.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:02.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:02.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:02.245 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:02.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:02.347 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:02.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:02.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:02.527 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:02.607 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 13 pg_num to 26 2024-09-14T04:27:02.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:02.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:02.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:02.807 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:02.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[107399]: pgmap v1432: 109 pgs: 109 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:02.838 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2198716032' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.838 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[112165]: pgmap v1432: 109 pgs: 109 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:02.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2198716032' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.838 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:02 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:02 smithi012 ceph-mon[101180]: pgmap v1432: 109 pgs: 109 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2198716032' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:02 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:27:02.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:02.941 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:02.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:03.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:03.073 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:03.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:03.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:03.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:03.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:03.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:03.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:03.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:03.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:03.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:03.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:03.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:03.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:03.779 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:03.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:03 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:27:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:03 smithi012 ceph-mon[101180]: osdmap e429: 8 total, 8 up, 6 in 2024-09-14T04:27:03.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:03 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:27:03.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:03.904 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:03.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:03.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:03.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:27:03.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[107399]: osdmap e429: 8 total, 8 up, 6 in 2024-09-14T04:27:03.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:27:03.959 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:27:03.959 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[112165]: osdmap e429: 8 total, 8 up, 6 in 2024-09-14T04:27:03.959 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:03 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:27:04.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:04.029 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:04.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:04.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:04.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:04.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:04.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:04.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:04.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:04.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:04.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:04.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:04.633 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:04.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:04.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:04.826 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:04.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:04 smithi012 ceph-mon[101180]: pgmap v1434: 109 pgs: 109 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:04.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:04 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:27:04.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:04 smithi012 ceph-mon[101180]: osdmap e430: 8 total, 8 up, 6 in 2024-09-14T04:27:04.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:04.943 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[107399]: pgmap v1434: 109 pgs: 109 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:04.943 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:27:04.943 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[107399]: osdmap e430: 8 total, 8 up, 6 in 2024-09-14T04:27:04.943 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[112165]: pgmap v1434: 109 pgs: 109 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:27:04.943 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:27:04.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:04 smithi001 ceph-mon[112165]: osdmap e430: 8 total, 8 up, 6 in 2024-09-14T04:27:04.952 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:27:04.952 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:27:04.986 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:04.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:04.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:05.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:05.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:05.140 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:27:05.140 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:27:05.179 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:27:05.179 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:27:05.212 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:27:05.213 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:27:05.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:05.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:05.258 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:27:05.259 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:27:05.295 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:27:05.295 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:27:05.328 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:27:05.328 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:27:05.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:05.361 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:27:05.362 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:27:05.394 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:27:05.395 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:27:05.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:05.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:05.631 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:05.689 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:05.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:27:05.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:05.831 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:05.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:05.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:05.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[107399]: osdmap e431: 8 total, 8 up, 6 in 2024-09-14T04:27:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1356013566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[107399]: 13.6 scrub starts 2024-09-14T04:27:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[112165]: osdmap e431: 8 total, 8 up, 6 in 2024-09-14T04:27:05.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:05.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1356013566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:05.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:05 smithi001 ceph-mon[112165]: 13.6 scrub starts 2024-09-14T04:27:05.988 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:06.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:06.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:05 smithi012 ceph-mon[101180]: osdmap e431: 8 total, 8 up, 6 in 2024-09-14T04:27:06.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:06.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1356013566' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:06.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:05 smithi012 ceph-mon[101180]: 13.6 scrub starts 2024-09-14T04:27:06.097 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:06.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:06.226 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:06.328 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:06.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:06.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:06.512 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:06.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:06.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:06.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:06.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:06.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:06.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:06.838 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: 13.0 scrub starts 2024-09-14T04:27:06.838 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: 13.8 scrub starts 2024-09-14T04:27:06.838 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: pgmap v1437: 119 pgs: 1 active+clean+scrubbing, 1 peering, 7 unknown, 110 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-14T04:27:06.839 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: 13.3 scrub starts 2024-09-14T04:27:06.839 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: 13.6 scrub ok 2024-09-14T04:27:06.839 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:06 smithi012 ceph-mon[101180]: 13.7 scrub starts 2024-09-14T04:27:06.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: 13.0 scrub starts 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: 13.8 scrub starts 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: pgmap v1437: 119 pgs: 1 active+clean+scrubbing, 1 peering, 7 unknown, 110 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: 13.3 scrub starts 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: 13.6 scrub ok 2024-09-14T04:27:06.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[112165]: 13.7 scrub starts 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: 13.0 scrub starts 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: 13.8 scrub starts 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: pgmap v1437: 119 pgs: 1 active+clean+scrubbing, 1 peering, 7 unknown, 110 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: 13.3 scrub starts 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: 13.6 scrub ok 2024-09-14T04:27:06.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-mon[107399]: 13.7 scrub starts 2024-09-14T04:27:06.917 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:06] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-14T04:27:06.930 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:06.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:07.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:07.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:07.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:07.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:07.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:07.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:07.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:07.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:07.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:07.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:07.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:07.661 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:27:07.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:07.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:07.783 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.4 scrub starts 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.10 scrub starts 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.0 scrub ok 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.8 scrub ok 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.3 scrub ok 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.13 scrub starts 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: 13.7 scrub ok 2024-09-14T04:27:07.784 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:07 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:07.818 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:27:07.819 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:27:07.819 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2024-09-14T04:27:07.819 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:27:07.859 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:07.896 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:07.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.4 scrub starts 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.10 scrub starts 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.0 scrub ok 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.8 scrub ok 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.3 scrub ok 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.13 scrub starts 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: 13.7 scrub ok 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:07.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.4 scrub starts 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.10 scrub starts 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.0 scrub ok 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.8 scrub ok 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.3 scrub ok 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.13 scrub starts 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: 13.7 scrub ok 2024-09-14T04:27:07.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:07 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:07.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:08.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:08.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:08.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:08.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:08.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:08.306 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:08.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:08.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:08.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:08.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:27:08.535 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:08.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:08.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:08.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:08.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: 13.10 scrub ok 2024-09-14T04:27:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: 13.1 scrub starts 2024-09-14T04:27:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: 13.4 scrub ok 2024-09-14T04:27:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: 13.13 scrub ok 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: pgmap v1438: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: osdmap e432: 8 total, 8 up, 6 in 2024-09-14T04:27:08.847 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:08 smithi012 ceph-mon[101180]: 13.2 scrub starts 2024-09-14T04:27:08.857 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:08.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:08.874 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:08.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:27:08.957 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: 13.10 scrub ok 2024-09-14T04:27:08.957 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: 13.1 scrub starts 2024-09-14T04:27:08.957 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: 13.4 scrub ok 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: 13.13 scrub ok 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: pgmap v1438: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: osdmap e432: 8 total, 8 up, 6 in 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[112165]: 13.2 scrub starts 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: 13.10 scrub ok 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: 13.1 scrub starts 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: 13.4 scrub ok 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: 13.13 scrub ok 2024-09-14T04:27:08.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: pgmap v1438: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:08.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:27:08.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:27:08.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2983378536' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:08.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: osdmap e432: 8 total, 8 up, 6 in 2024-09-14T04:27:08.959 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:08 smithi001 ceph-mon[107399]: 13.2 scrub starts 2024-09-14T04:27:08.959 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:09.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:09.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:09.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:09.310 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:09.392 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:27:09.392 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:27:09.393 INFO:tasks.thrashosds.thrasher:Adding osd 7 2024-09-14T04:27:09.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd in 7 2024-09-14T04:27:09.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:09.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:09.595 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:09.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.1 scrub ok 2024-09-14T04:27:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.11 scrub starts 2024-09-14T04:27:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.2 scrub ok 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.11 scrub ok 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.5 scrub starts 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[112165]: 13.9 deep-scrub starts 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.1 scrub ok 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.11 scrub starts 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.2 scrub ok 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.11 scrub ok 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.5 scrub starts 2024-09-14T04:27:09.722 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:09 smithi001 ceph-mon[107399]: 13.9 deep-scrub starts 2024-09-14T04:27:09.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:09.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:09.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:09.934 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:09.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:10.004 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.1 scrub ok 2024-09-14T04:27:10.004 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.11 scrub starts 2024-09-14T04:27:10.004 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.2 scrub ok 2024-09-14T04:27:10.005 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.11 scrub ok 2024-09-14T04:27:10.005 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.5 scrub starts 2024-09-14T04:27:10.005 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:09 smithi012 ceph-mon[101180]: 13.9 deep-scrub starts 2024-09-14T04:27:10.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:10.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:10.180 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:10.183 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:10.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:10.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:10.371 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:10.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:10.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:10.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:10.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:10.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:10.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:10.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:27:10.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:10.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:10.892 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:10.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[107399]: 13.18 scrub starts 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[107399]: pgmap v1440: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[107399]: 13.5 scrub ok 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[112165]: 13.18 scrub starts 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[112165]: pgmap v1440: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:10 smithi001 ceph-mon[112165]: 13.5 scrub ok 2024-09-14T04:27:10.982 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:10 smithi012 ceph-mon[101180]: 13.18 scrub starts 2024-09-14T04:27:10.982 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:10 smithi012 ceph-mon[101180]: pgmap v1440: 119 pgs: 3 active+clean+scrubbing, 1 peering, 1 unknown, 114 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T04:27:10.982 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:10 smithi012 ceph-mon[101180]: 13.5 scrub ok 2024-09-14T04:27:10.994 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:11.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:11.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:11.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:11.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:11.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:11.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:11.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:11.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:11.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:11.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:11.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:11.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:11.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:11.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:11.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[112165]: 13.18 scrub ok 2024-09-14T04:27:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[112165]: 13.14 scrub starts 2024-09-14T04:27:11.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[112165]: 13.15 scrub starts 2024-09-14T04:27:11.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[112165]: pgmap v1441: 119 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 117 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:11.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T04:27:11.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[107399]: 13.18 scrub ok 2024-09-14T04:27:11.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[107399]: 13.14 scrub starts 2024-09-14T04:27:11.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[107399]: 13.15 scrub starts 2024-09-14T04:27:11.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[107399]: pgmap v1441: 119 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 117 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:11.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:11 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T04:27:11.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:12.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:11 smithi012 ceph-mon[101180]: 13.18 scrub ok 2024-09-14T04:27:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:11 smithi012 ceph-mon[101180]: 13.14 scrub starts 2024-09-14T04:27:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:11 smithi012 ceph-mon[101180]: 13.15 scrub starts 2024-09-14T04:27:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:11 smithi012 ceph-mon[101180]: pgmap v1441: 119 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 117 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:12.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:11 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T04:27:12.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:12.144 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:12.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:12.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:12.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:12.413 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:12.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:12.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:12.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:12.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:12.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:12.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:12.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: 13.14 scrub ok 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: 13.15 scrub ok 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: osdmap e433: 8 total, 8 up, 6 in 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: 13.17 scrub starts 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: 13.12 scrub starts 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1660006010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:12.898 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: 13.14 scrub ok 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: 13.15 scrub ok 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: osdmap e433: 8 total, 8 up, 6 in 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: 13.17 scrub starts 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: 13.12 scrub starts 2024-09-14T04:27:12.899 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1660006010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:12.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:12.925 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:12.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:13.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:13.046 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: 13.14 scrub ok 2024-09-14T04:27:13.046 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: 13.15 scrub ok 2024-09-14T04:27:13.046 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T04:27:13.047 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: osdmap e433: 8 total, 8 up, 6 in 2024-09-14T04:27:13.047 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: 13.17 scrub starts 2024-09-14T04:27:13.047 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: 13.12 scrub starts 2024-09-14T04:27:13.047 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1660006010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:13.052 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:13.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:13.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:13.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:13.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:13.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:13.325 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:27:13.325 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:27:13.380 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:27:13.380 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:27:13.380 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:27:13.380 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:27:13.381 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:27:13.382 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:27:13.382 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:27:13.382 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:27:13.382 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:27:13.382 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:27:13.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:13.430 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:13.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:13.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:13.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:13.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:13.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:13.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:13.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:13.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:13.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:13.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:13.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[112165]: 13.17 scrub ok 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[112165]: 13.12 scrub ok 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[112165]: osdmap e434: 8 total, 8 up, 6 in 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[112165]: pgmap v1444: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[107399]: 13.17 scrub ok 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[107399]: 13.12 scrub ok 2024-09-14T04:27:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[107399]: osdmap e434: 8 total, 8 up, 6 in 2024-09-14T04:27:13.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[107399]: pgmap v1444: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:13.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:13 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T04:27:13.976 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:13 smithi012 ceph-mon[101180]: 13.17 scrub ok 2024-09-14T04:27:13.976 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:13 smithi012 ceph-mon[101180]: 13.12 scrub ok 2024-09-14T04:27:13.976 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:13 smithi012 ceph-mon[101180]: osdmap e434: 8 total, 8 up, 6 in 2024-09-14T04:27:13.976 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:13 smithi012 ceph-mon[101180]: pgmap v1444: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:13.976 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:13 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T04:27:14.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:27:14.066 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:14.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:14.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:14.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:14.321 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:14.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:14.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:14.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:14.495 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:14.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:14.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:14.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:14.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:14 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T04:27:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:14 smithi012 ceph-mon[101180]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 432'9382 2024-09-14T04:27:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:14 smithi012 ceph-mon[101180]: osdmap e435: 8 total, 8 up, 6 in 2024-09-14T04:27:14.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:14 smithi012 ceph-mon[101180]: osdmap e436: 8 total, 8 up, 6 in 2024-09-14T04:27:14.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:14.874 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:14.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:14.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:14.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[107399]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 432'9382 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[107399]: osdmap e435: 8 total, 8 up, 6 in 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[107399]: osdmap e436: 8 total, 8 up, 6 in 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[112165]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 432'9382 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[112165]: osdmap e435: 8 total, 8 up, 6 in 2024-09-14T04:27:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:14 smithi001 ceph-mon[112165]: osdmap e436: 8 total, 8 up, 6 in 2024-09-14T04:27:14.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:15.003 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:15.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:15.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:15.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:15.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:15.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:15.404 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:15.506 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:15.544 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:15.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:15.671 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:15.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:15.776 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:15.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:15.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:15.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:15.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:16.058 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:16.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:16.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:16.227 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:16.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:16.359 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:16.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:16.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:16 smithi012 ceph-mon[101180]: osdmap e437: 8 total, 8 up, 6 in 2024-09-14T04:27:16.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:16 smithi012 ceph-mon[101180]: 13.11 starting backfill to osd.1 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:16 smithi012 ceph-mon[101180]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:16 smithi012 ceph-mon[101180]: pgmap v1448: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:16.431 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T04:27:16.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:16.492 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:16.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:16.622 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[107399]: osdmap e437: 8 total, 8 up, 6 in 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[107399]: 13.11 starting backfill to osd.1 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[107399]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[107399]: pgmap v1448: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[112165]: osdmap e437: 8 total, 8 up, 6 in 2024-09-14T04:27:16.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[112165]: 13.11 starting backfill to osd.1 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[112165]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 433'9098 2024-09-14T04:27:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[112165]: pgmap v1448: 119 pgs: 1 active+clean+scrubbing+deep, 118 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail 2024-09-14T04:27:16.705 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T04:27:16.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:16.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:16.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:16.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:16.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:16] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-14T04:27:16.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:17.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:17.104 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:17.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:17.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:17.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:17.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:17.381 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:17.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:17.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:17 smithi012 ceph-mon[101180]: 13.16 scrub starts 2024-09-14T04:27:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T04:27:17.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:17 smithi012 ceph-mon[101180]: osdmap e438: 8 total, 8 up, 6 in 2024-09-14T04:27:17.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:17.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:17.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:17.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:27:17.666 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[107399]: 13.16 scrub starts 2024-09-14T04:27:17.666 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T04:27:17.667 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[107399]: osdmap e438: 8 total, 8 up, 6 in 2024-09-14T04:27:17.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[112165]: 13.16 scrub starts 2024-09-14T04:27:17.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T04:27:17.667 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:17 smithi001 ceph-mon[112165]: osdmap e438: 8 total, 8 up, 6 in 2024-09-14T04:27:17.734 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:17.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:17.861 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:17.884 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:17.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:17.986 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:18.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:18.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:18.134 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:18.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:18.260 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:18.362 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:18.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:18.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:18.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:18 smithi012 ceph-mon[101180]: 13.9 deep-scrub ok 2024-09-14T04:27:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:18 smithi012 ceph-mon[101180]: osdmap e439: 8 total, 8 up, 6 in 2024-09-14T04:27:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:18 smithi012 ceph-mon[101180]: pgmap v1451: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:18.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-14T04:27:18.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:18.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[107399]: 13.9 deep-scrub ok 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[107399]: osdmap e439: 8 total, 8 up, 6 in 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[107399]: pgmap v1451: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[112165]: 13.9 deep-scrub ok 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[112165]: osdmap e439: 8 total, 8 up, 6 in 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[112165]: pgmap v1451: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-14T04:27:18.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:18.805 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:18.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:18.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:18.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:19.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:19.057 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:19.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:19.159 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:19.164 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:27:19.164 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:27:19.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:19.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:19.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:19.312 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:19.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:19.437 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:19.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: osdmap e440: 8 total, 8 up, 6 in 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: 13.12 starting backfill to osd.0 from (0'0,412'3094] MIN to 437'9386 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 437'9386 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3875892817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[112165]: osdmap e441: 8 total, 8 up, 7 in 2024-09-14T04:27:19.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:27:19.130+0000 7fbfffcba640 -1 mon.a@0(leader).osd e440 definitely_dead 0 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: osdmap e440: 8 total, 8 up, 6 in 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: 13.12 starting backfill to osd.0 from (0'0,412'3094] MIN to 437'9386 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 437'9386 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3875892817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-14T04:27:19.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:19 smithi001 ceph-mon[107399]: osdmap e441: 8 total, 8 up, 7 in 2024-09-14T04:27:19.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.563 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:19.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-14T04:27:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: osdmap e440: 8 total, 8 up, 6 in 2024-09-14T04:27:19.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: 13.12 starting backfill to osd.0 from (0'0,412'3094] MIN to 437'9386 2024-09-14T04:27:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 437'9386 2024-09-14T04:27:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3875892817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-14T04:27:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-14T04:27:19.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:19 smithi012 ceph-mon[101180]: osdmap e441: 8 total, 8 up, 7 in 2024-09-14T04:27:19.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:19.688 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:19.704 INFO:teuthology.orchestra.run.smithi001.stderr:marked in osd.7. 2024-09-14T04:27:19.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:19.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:19.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:19.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:19.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:20.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:27:20.045 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:20.070 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:20.172 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:20.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:20.225 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:20.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.322 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:20.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:20.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:20.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:20.508 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: 13.16 scrub starts 2024-09-14T04:27:20.508 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: pgmap v1454: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:20.508 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:27:20.509 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:20.509 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:20.509 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:27:20.509 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:20 smithi012 ceph-mon[101180]: osdmap e442: 8 total, 8 up, 7 in 2024-09-14T04:27:20.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:20.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:20.651 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:20.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:20.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: 13.16 scrub starts 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: pgmap v1454: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[107399]: osdmap e442: 8 total, 8 up, 7 in 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: 13.16 scrub starts 2024-09-14T04:27:20.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: pgmap v1454: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 116 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 6426/242688 objects misplaced (2.648%) 2024-09-14T04:27:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T04:27:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T04:27:20.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:20 smithi001 ceph-mon[112165]: osdmap e442: 8 total, 8 up, 7 in 2024-09-14T04:27:20.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:20.825 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:20.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:20.951 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:20.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:21.052 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:21.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:21.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:21.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:21.209 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:21.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:21.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:21.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[107399]: 13.16 deep-scrub starts 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[107399]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[107399]: 13.13 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[107399]: osdmap e443: 8 total, 8 up, 7 in 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[112165]: 13.16 deep-scrub starts 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[112165]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[112165]: 13.13 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:21 smithi001 ceph-mon[112165]: osdmap e443: 8 total, 8 up, 7 in 2024-09-14T04:27:21.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:21.515 INFO:tasks.thrashosds.thrasher:Added osd 7 2024-09-14T04:27:21.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:21 smithi012 ceph-mon[101180]: 13.16 deep-scrub starts 2024-09-14T04:27:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:21 smithi012 ceph-mon[101180]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:21 smithi012 ceph-mon[101180]: 13.13 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9484 2024-09-14T04:27:21.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:21 smithi012 ceph-mon[101180]: osdmap e443: 8 total, 8 up, 7 in 2024-09-14T04:27:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.612 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:21.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:21.713 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:21.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:21.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:21.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:21.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:21.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:21.984 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:22.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:22.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:22.143 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:22.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:22.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.d starting backfill to osd.7 from (0'0,0'0] MAX to 440'9868 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.1 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9166 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.e starting backfill to osd.7 from (0'0,0'0] MAX to 440'9675 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.7 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9546 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.f starting backfill to osd.7 from (0'0,0'0] MAX to 440'9712 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.0 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9509 2024-09-14T04:27:22.282 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: 13.17 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9571 2024-09-14T04:27:22.283 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: pgmap v1457: 119 pgs: 21 peering, 1 active+remapped+backfill_wait, 4 remapped+peering, 9 unknown, 1 active+clean+scrubbing+deep, 83 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4854/187755 objects misplaced (2.585%); 24 MiB/s, 389 objects/s recovering 2024-09-14T04:27:22.283 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:22 smithi012 ceph-mon[101180]: osdmap e444: 8 total, 8 up, 7 in 2024-09-14T04:27:22.305 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:27:22.306 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:27:22.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:22.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.d starting backfill to osd.7 from (0'0,0'0] MAX to 440'9868 2024-09-14T04:27:22.385 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.1 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9166 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.e starting backfill to osd.7 from (0'0,0'0] MAX to 440'9675 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.7 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9546 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.f starting backfill to osd.7 from (0'0,0'0] MAX to 440'9712 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.0 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9509 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: 13.17 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9571 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: pgmap v1457: 119 pgs: 21 peering, 1 active+remapped+backfill_wait, 4 remapped+peering, 9 unknown, 1 active+clean+scrubbing+deep, 83 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4854/187755 objects misplaced (2.585%); 24 MiB/s, 389 objects/s recovering 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[112165]: osdmap e444: 8 total, 8 up, 7 in 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.d starting backfill to osd.7 from (0'0,0'0] MAX to 440'9868 2024-09-14T04:27:22.386 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.1 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9166 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.e starting backfill to osd.7 from (0'0,0'0] MAX to 440'9675 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.7 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9546 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.f starting backfill to osd.7 from (0'0,0'0] MAX to 440'9712 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.0 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9509 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: 13.17 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9571 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: pgmap v1457: 119 pgs: 21 peering, 1 active+remapped+backfill_wait, 4 remapped+peering, 9 unknown, 1 active+clean+scrubbing+deep, 83 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4854/187755 objects misplaced (2.585%); 24 MiB/s, 389 objects/s recovering 2024-09-14T04:27:22.387 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:22 smithi001 ceph-mon[107399]: osdmap e444: 8 total, 8 up, 7 in 2024-09-14T04:27:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:22.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:22.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:22.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:22.656 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.700 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:22.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:22.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:22.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:22.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:22.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:27:22.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:22.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:22.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:23.094 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:23.133 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:23.235 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:23.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:23 smithi012 ceph-mon[101180]: 13.19 scrub starts 2024-09-14T04:27:23.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:23 smithi012 ceph-mon[101180]: 13.14 starting backfill to osd.0 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:23 smithi012 ceph-mon[101180]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:23 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 9 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-09-14T04:27:23.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:23 smithi012 ceph-mon[101180]: osdmap e445: 8 total, 8 up, 7 in 2024-09-14T04:27:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.279 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:23.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:23.336 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:23.380 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.428 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:23.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:23.529 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:23.567 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:23.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:23.706 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:23.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[107399]: 13.19 scrub starts 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[107399]: 13.14 starting backfill to osd.0 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[107399]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 9 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[107399]: osdmap e445: 8 total, 8 up, 7 in 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[112165]: 13.19 scrub starts 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[112165]: 13.14 starting backfill to osd.0 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[112165]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 440'9606 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 9 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-09-14T04:27:23.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:23 smithi001 ceph-mon[112165]: osdmap e445: 8 total, 8 up, 7 in 2024-09-14T04:27:23.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:23.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:23.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:23.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:23.987 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:23.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:24.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:24.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:24.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:24.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:24.318 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:24 smithi001 ceph-mon[112165]: 13.19 scrub ok 2024-09-14T04:27:24.318 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:24 smithi001 ceph-mon[112165]: pgmap v1460: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 22 peering, 2 active+remapped+backfill_wait, 4 remapped+peering, 8 unknown, 1 active+clean+scrubbing+deep, 78 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 14628/201714 objects misplaced (7.252%); 44 MiB/s, 703 objects/s recovering 2024-09-14T04:27:24.319 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:24 smithi001 ceph-mon[107399]: 13.19 scrub ok 2024-09-14T04:27:24.319 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:24 smithi001 ceph-mon[107399]: pgmap v1460: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 22 peering, 2 active+remapped+backfill_wait, 4 remapped+peering, 8 unknown, 1 active+clean+scrubbing+deep, 78 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 14628/201714 objects misplaced (7.252%); 44 MiB/s, 703 objects/s recovering 2024-09-14T04:27:24.371 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:24.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:24.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:24.536 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:27:24.537 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:27:24.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:24.557 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:27:24.557 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:27:24.595 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:27:24.595 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:27:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:24 smithi012 ceph-mon[101180]: 13.19 scrub ok 2024-09-14T04:27:24.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:24 smithi012 ceph-mon[101180]: pgmap v1460: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 22 peering, 2 active+remapped+backfill_wait, 4 remapped+peering, 8 unknown, 1 active+clean+scrubbing+deep, 78 active+clean; 4.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 14628/201714 objects misplaced (7.252%); 44 MiB/s, 703 objects/s recovering 2024-09-14T04:27:24.631 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:27:24.631 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:27:24.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:24.667 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:27:24.667 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:27:24.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:24.703 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:27:24.703 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:27:24.747 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:27:24.747 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:27:24.789 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:27:24.789 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:27:24.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:24.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:24.926 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:24.959 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:24.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:25.060 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:25.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:25.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:25.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:25.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:25.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:25.384 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:25.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:25.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:25.492 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:27:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:25.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:25 smithi012 ceph-mon[101180]: 13.16 scrub starts 2024-09-14T04:27:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/755223303' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:25 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2040447779' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:25 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.618 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:25.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[107399]: 13.16 scrub starts 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/755223303' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2040447779' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[112165]: 13.16 scrub starts 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/755223303' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2040447779' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:25 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:27:25.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:25.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:25.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:25.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:25.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:25.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:26.036 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:27:26.036 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:27:26.036 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:27:26.036 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:27:26.036 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:27:26.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:26.072 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:26.174 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:26.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:26.349 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:26.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:26.491 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:26.516 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 7] out_osds: [6] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:27:26.516 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:27:26.516 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2024-09-14T04:27:26.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[107399]: 13.16 scrub ok 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[107399]: osdmap e446: 8 total, 8 up, 7 in 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[107399]: pgmap v1462: 119 pgs: 2 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 1 peering, 7 active+remapped+backfill_wait, 2 remapped+peering, 104 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 159 MiB/s wr, 3.92k op/s; 34036/234174 objects misplaced (14.534%); 33 MiB/s, 2 keys/s, 532 objects/s recovering 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[112165]: 13.16 scrub ok 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:27:26.573 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[112165]: osdmap e446: 8 total, 8 up, 7 in 2024-09-14T04:27:26.574 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-mon[112165]: pgmap v1462: 119 pgs: 2 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 1 peering, 7 active+remapped+backfill_wait, 2 remapped+peering, 104 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 159 MiB/s wr, 3.92k op/s; 34036/234174 objects misplaced (14.534%); 33 MiB/s, 2 keys/s, 532 objects/s recovering 2024-09-14T04:27:26.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:26 smithi012 ceph-mon[101180]: 13.16 scrub ok 2024-09-14T04:27:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:26 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:26 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:27:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:26 smithi012 ceph-mon[101180]: osdmap e446: 8 total, 8 up, 7 in 2024-09-14T04:27:26.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:26 smithi012 ceph-mon[101180]: pgmap v1462: 119 pgs: 2 active+remapped+backfilling, 2 active+undersized+remapped, 1 active+remapped, 1 peering, 7 active+remapped+backfill_wait, 2 remapped+peering, 104 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 159 MiB/s wr, 3.92k op/s; 34036/234174 objects misplaced (14.534%); 33 MiB/s, 2 keys/s, 532 objects/s recovering 2024-09-14T04:27:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:26.758 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:26.809 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:26.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:26.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:26.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:26] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.51.0" 2024-09-14T04:27:27.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:27.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:27.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:27.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:27.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:27.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:27:27.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:27.399 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:27.501 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:27.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:27.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:27 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-09-14T04:27:27.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:27.655 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:27:27.661 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:27 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-09-14T04:27:27.662 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:27 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-09-14T04:27:27.686 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:27.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:27.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:27.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:27.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:27.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:28.054 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:28.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:28.112 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:28.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:28.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:28.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:28.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:28.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:28.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:28.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:28 smithi012 ceph-mon[101180]: pgmap v1463: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 76 MiB/s wr, 2.54k op/s; 36477/224646 objects misplaced (16.238%); 30 MiB/s, 1 keys/s, 483 objects/s recovering 2024-09-14T04:27:28.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:28.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:28.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:28.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:28 smithi001 ceph-mon[107399]: pgmap v1463: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 76 MiB/s wr, 2.54k op/s; 36477/224646 objects misplaced (16.238%); 30 MiB/s, 1 keys/s, 483 objects/s recovering 2024-09-14T04:27:28.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:28 smithi001 ceph-mon[112165]: pgmap v1463: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 76 MiB/s wr, 2.54k op/s; 36477/224646 objects misplaced (16.238%); 30 MiB/s, 1 keys/s, 483 objects/s recovering 2024-09-14T04:27:28.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.806 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:28.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:28.907 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:28.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:28.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:29.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:29.085 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:29.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:29.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:29.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:29.327 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:29.370 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:29.472 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:29.480 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:27:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.513 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:29.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:29.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.648 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145636 12.d 12:b54dcd2d:::benchmark_data_smithi001_255951_object145635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.21317292700002, 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6968347069999998, 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145636 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431912+0000", 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431910+0000", 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:27:29.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431939+0000", 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8583000000000001e-05 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431948+0000", 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2059999999999996e-06 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.431954+0000", 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6269999999999997e-06 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432030+0000", 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5821000000000005e-05 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432175+0000", 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014539199999999999 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.432308+0000", 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013317900000000001 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.441523+0000", 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092149239999999993 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.449041+0000", 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00751791 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.449062+0000", 2024-09-14T04:27:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0616999999999999e-05 2024-09-14T04:27:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434784+0000", 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98572267800000002 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434800+0000", 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5634e-05 2024-09-14T04:27:29.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128682+0000", 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6938822039999999 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128746+0000", 2024-09-14T04:27:29.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4221999999999996e-05 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145654 12.d 12:b3f58279:::benchmark_data_smithi001_255951_object145653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.17638784100001, 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.664128394, 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.658 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145654 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468697+0000", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468695+0000", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468737+0000", 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1696000000000002e-05 2024-09-14T04:27:29.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468747+0000", 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8840000000000006e-06 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468752+0000", 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2320000000000001e-06 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468800+0000", 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8006000000000001e-05 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.468807+0000", 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7009999999999999e-06 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434836+0000", 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96602920299999995 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.434846+0000", 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0328e-05 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.128937+0000", 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6940900490000002 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.129068+0000", 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013180099999999999 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.129192+0000", 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012386299999999999 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132186+0000", 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029939110000000001 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132344+0000", 2024-09-14T04:27:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157539 2024-09-14T04:27:29.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132719+0000", 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037569300000000002 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132757+0000", 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7171e-05 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:49.132825+0000", 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8707999999999993e-05 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:145656 12.7 12:e2152f9f:::benchmark_data_smithi001_255951_object145655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.17356702799998, 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4715065950000001, 2024-09-14T04:27:29.668 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 145656 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471518+0000", 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471516+0000", 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471570+0000", 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3325999999999997e-05 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471581+0000", 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1319e-05 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471588+0000", 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7089999999999996e-06 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471626+0000", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7920000000000003e-05 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:46.471633+0000", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0330000000000003e-06 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.435398+0000", 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96376509200000005 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:47.435407+0000", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4560000000000006e-06 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939054+0000", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5036471769999999 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939205+0000", 2024-09-14T04:27:29.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015092799999999999 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.939321+0000", 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116131 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942283+0000", 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029620380000000002 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942614+0000", 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000330861 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942930+0000", 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031613099999999999 2024-09-14T04:27:29.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.942967+0000", 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6519000000000001e-05 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:19:48.943024+0000", 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7374999999999999e-05 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169693 12.e e301/283 12:7fb05d18:::benchmark_data_smithi001_255951_object24033:head v 301'10537, mlcod=301'10533)", 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.18925154499999, 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38418591800000002, 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:27:29.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455833+0000", 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455836+0000", 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4510000000000001e-06 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455843+0000", 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4989999999999999e-06 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.455846+0000", 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.569e-06 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.465968+0000", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010121404000000001 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.465987+0000", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9069e-05 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838217+0000", 2024-09-14T04:27:29.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37223062200000001 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840019+0000", 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00180157 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169704 12.2 e301/283 12:4868bfdc:::benchmark_data_smithi001_255951_object24044:head v 301'10684, mlcod=301'10682)", 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.18135352399997, 2024-09-14T04:27:29.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.376235192, 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463731+0000", 2024-09-14T04:27:29.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463730+0000", 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463738+0000", 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463749+0000", 2024-09-14T04:27:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0523000000000001e-05 2024-09-14T04:27:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463755+0000", 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2199999999999997e-06 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463795+0000", 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0185999999999998e-05 2024-09-14T04:27:29.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.463813+0000", 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8094999999999999e-05 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838212+0000", 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37439889199999998 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.839966+0000", 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017544469999999999 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169707 12.0 e301/298 12:0f3a37da:::benchmark_data_smithi001_255951_object24047:head v 301'10515, mlcod=301'10513)", 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.18042293500002, 2024-09-14T04:27:29.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37545621800000001, 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464662+0000", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464661+0000", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464667+0000", 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0279999999999997e-06 2024-09-14T04:27:29.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464686+0000", 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8640000000000001e-05 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464692+0000", 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8150000000000002e-06 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464725+0000", 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3404000000000001e-05 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.464746+0000", 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0526999999999998e-05 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838191+0000", 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.373445845 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840118+0000", 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001926599 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169710 12.2 e301/283 12:4ac1415e:::benchmark_data_smithi001_255951_object24050:head v 301'10685, mlcod=301'10682)", 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.17761867799999, 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37253622600000003, 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.692 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467466+0000", 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467465+0000", 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467474+0000", 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2029999999999998e-06 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467486+0000", 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1875999999999999e-05 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467493+0000", 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6610000000000003e-06 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467530+0000", 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7261000000000001e-05 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467548+0000", 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8037999999999999e-05 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.839973+0000", 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37242533 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840002+0000", 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8946999999999999e-05 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169711 12.0 e301/298 12:0996da4d:::benchmark_data_smithi001_255951_object24051:head v 301'10516, mlcod=301'10513)", 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.17708462799999, 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37581841599999999, 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.697 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468000+0000", 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.467999+0000", 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468006+0000", 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1339999999999998e-06 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468016+0000", 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.026e-05 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468023+0000", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3439999999999997e-06 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468100+0000", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6761999999999995e-05 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468120+0000", 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0733e-05 2024-09-14T04:27:29.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.843700+0000", 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37557949099999999 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.843818+0000", 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118651 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.701 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:169713 12.7 12:ec3b0951:::benchmark_data_smithi001_255951_object24053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.17653567600001, 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38481411700000001, 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.702 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 169713 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468549+0000", 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468547+0000", 2024-09-14T04:27:29.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468554+0000", 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4729999999999997e-06 2024-09-14T04:27:29.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468567+0000", 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3057000000000001e-05 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.468574+0000", 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2420000000000003e-06 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.469065+0000", 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00049128599999999996 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.469183+0000", 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117336 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840086+0000", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37090367000000002 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.851198+0000", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011112260000000001 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.851461+0000", 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026300800000000001 2024-09-14T04:27:29.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853194+0000", 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001732555 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853268+0000", 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3950999999999997e-05 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.853363+0000", 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5117000000000004e-05 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169716 12.1 e301/283 12:8a7b11ab:::benchmark_data_smithi001_255951_object24056:head v 301'10620, mlcod=301'10618)", 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.17397815800001, 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38175547799999998, 2024-09-14T04:27:29.712 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471107+0000", 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471106+0000", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471113+0000", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.486e-06 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838195+0000", 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.367081775 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838211+0000", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5896000000000002e-05 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838254+0000", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2775000000000003e-05 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838278+0000", 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4409e-05 2024-09-14T04:27:29.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.852789+0000", 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014511082 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.852862+0000", 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2694000000000006e-05 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:29.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:29.718 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45647.0:169712 12.0 e301/298 12:07f327ad:::benchmark_data_smithi001_255951_object24052:head v 301'10517, mlcod=301'10513)", 2024-09-14T04:27:29.718 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:27:29.718 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 432.17325897199999, 2024-09-14T04:27:29.718 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37244635700000001, 2024-09-14T04:27:29.718 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471826+0000", 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471825+0000", 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.471832+0000", 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2640000000000002e-06 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838208+0000", 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.36637539400000002 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.838216+0000", 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.67e-06 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840153+0000", 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019370069999999999 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.840173+0000", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0157e-05 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.844245+0000", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040714260000000004 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:20:17.844272+0000", 2024-09-14T04:27:29.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7202000000000001e-05 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:242450 12.d 12:b7679709:::benchmark_data_smithi001_255951_object96790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 326.050483493, 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.28077718200000001, 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.723 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 242450 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594601+0000", 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594600+0000", 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594606+0000", 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5810000000000002e-06 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594621+0000", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5125e-05 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594629+0000", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9370000000000004e-06 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594687+0000", 2024-09-14T04:27:29.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8149000000000002e-05 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594834+0000", 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014639899999999999 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.594990+0000", 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015630400000000001 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.602164+0000", 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071733379999999996 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.602322+0000", 2024-09-14T04:27:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000158171 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875247+0000", 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.27292567000000001 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875288+0000", 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0624999999999998e-05 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:03.875378+0000", 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0378999999999996e-05 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:243548 12.7 12:ed83732d:::benchmark_data_smithi001_255951_object97888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 319.24335109499998, 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96499681999999998, 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.730 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 243548 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401731+0000", 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.732 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401744+0000", 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2456999999999999e-05 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401774+0000", 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0086e-05 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401787+0000", 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3644e-05 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401910+0000", 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012274000000000001 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401918+0000", 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.361975+0000", 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96005681700000001 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362097+0000", 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012235999999999999 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362195+0000", 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8171000000000006e-05 2024-09-14T04:27:29.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.365736+0000", 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035408940000000002 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366255+0000", 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051849299999999997 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366541+0000", 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028677599999999998 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366603+0000", 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2151999999999997e-05 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366730+0000", 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012696499999999999 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:163932 13.f e446/442 13:fcf34562:::benchmark_data_smithi001_306460_object45688:head v 446'10144, mlcod=446'10142)", 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:27.936890+0000", 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.708194507, 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013849398000000001, 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.738 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936890+0000", 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936890+0000", 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936889+0000", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936897+0000", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4739999999999999e-06 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936913+0000", 2024-09-14T04:27:29.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5950000000000001e-05 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936919+0000", 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8650000000000003e-06 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.936987+0000", 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7179000000000002e-05 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.937008+0000", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1783000000000001e-05 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.950592+0000", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013583715 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.950740+0000", 2024-09-14T04:27:29.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014752600000000001 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:163945 13.7 e446/442 13:e02001cd:::benchmark_data_smithi001_306460_object45701:head v 446'9781, mlcod=446'9779)", 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:27.944237+0000", 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.700847607, 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015359128, 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.743 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944237+0000", 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944237+0000", 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944236+0000", 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944243+0000", 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3180000000000003e-06 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944253+0000", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0706999999999999e-05 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.944260+0000", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4110000000000001e-06 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.946909+0000", 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026493049999999998 2024-09-14T04:27:29.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.946929+0000", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9689000000000001e-05 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.959488+0000", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012559163 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.959596+0000", 2024-09-14T04:27:29.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010822199999999999 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:163971 13.e e446/442 13:748524d0:::benchmark_data_smithi001_306460_object45727:head v 446'10161, mlcod=446'10159)", 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:27.965394+0000", 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.6796902760000001, 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013819097000000001, 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965394+0000", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965394+0000", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965394+0000", 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965399+0000", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7679999999999999e-06 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965405+0000", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5230000000000001e-06 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965410+0000", 2024-09-14T04:27:29.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8509999999999998e-06 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965560+0000", 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014986099999999999 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.965580+0000", 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0273999999999999e-05 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.979175+0000", 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013594675000000001 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:27.979214+0000", 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8452999999999999e-05 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:164621 13.e e446/442 13:72a010c0:::benchmark_data_smithi001_306460_object46377:head v 446'10202, mlcod=446'10200)", 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:28.504917+0000", 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.140167978, 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014090699, 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504917+0000", 2024-09-14T04:27:29.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504917+0000", 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504916+0000", 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504926+0000", 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.009e-05 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504947+0000", 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1488000000000002e-05 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.504955+0000", 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1799999999999996e-06 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.505030+0000", 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5073000000000004e-05 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.505053+0000", 2024-09-14T04:27:29.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.281e-05 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.518878+0000", 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01382423 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.519007+0000", 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012986900000000001 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.757 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:164804 13.f e446/442 13:f22dcb90:::benchmark_data_smithi001_306460_object46560:head v 446'10195, mlcod=446'10193)", 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:28.649980+0000", 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.99510444899999995, 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017758426000000001, 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.649980+0000", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.649980+0000", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.649979+0000", 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.649985+0000", 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0260000000000004e-06 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.649997+0000", 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1598e-05 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.650004+0000", 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8970000000000002e-06 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.650062+0000", 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8019999999999997e-05 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.650082+0000", 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0293999999999999e-05 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.667690+0000", 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017607278000000001 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.667739+0000", 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9103000000000001e-05 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:164809 13.f e446/442 13:fc65fbc2:::benchmark_data_smithi001_306460_object46565:head v 446'10197, mlcod=446'10193)", 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:28.657727+0000", 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.98735813800000005, 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021472156999999999, 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657727+0000", 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657727+0000", 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657726+0000", 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657732+0000", 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1460000000000001e-06 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657748+0000", 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5831000000000001e-05 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.657754+0000", 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2430000000000002e-06 2024-09-14T04:27:29.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.658760+0000", 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001005354 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.658778+0000", 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8929000000000001e-05 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.679133+0000", 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020354537999999998 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.679199+0000", 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5730999999999998e-05 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.767 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46076.0:164812 13.f e446/442 13:f0a35d80:::benchmark_data_smithi001_306460_object46568:head v 446'10198, mlcod=446'10193)", 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:27:28.662163+0000", 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.98292128400000001, 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017087979, 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:27:29.768 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:27:29.769 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:27:29.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:27:29.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662163+0000", 2024-09-14T04:27:29.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662163+0000", 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662162+0000", 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:27:29.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662171+0000", 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1780000000000003e-06 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662188+0000", 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.755e-05 2024-09-14T04:27:29.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.662197+0000", 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.918e-06 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:27:29.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.664771+0000", 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025737059999999998 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.664791+0000", 2024-09-14T04:27:29.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0268999999999999e-05 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.679203+0000", 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014411767000000001 2024-09-14T04:27:29.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:27:28.679251+0000", 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8560000000000003e-05 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.777 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.778 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:27:29.778 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:27:29.778 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:27:29.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:29.863 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:29.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:29.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:29.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:30.036 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:27:30.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:30.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:30.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:30.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:30.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:30.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:30.430 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:30.450 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:30.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:30.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:30.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:30.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:30.734 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:30.790 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:30 smithi001 ceph-mon[107399]: pgmap v1464: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 2.08k op/s; 36477/224646 objects misplaced (16.238%); 20 MiB/s, 1 keys/s, 323 objects/s recovering 2024-09-14T04:27:30.790 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:30 smithi001 ceph-mon[112165]: pgmap v1464: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 2.08k op/s; 36477/224646 objects misplaced (16.238%); 20 MiB/s, 1 keys/s, 323 objects/s recovering 2024-09-14T04:27:30.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:30.883 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:30.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:30.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:31.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:31.028 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:31.075 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:27:31.077 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:27:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:30 smithi012 ceph-mon[101180]: pgmap v1464: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 62 MiB/s wr, 2.08k op/s; 36477/224646 objects misplaced (16.238%); 20 MiB/s, 1 keys/s, 323 objects/s recovering 2024-09-14T04:27:31.116 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:31.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.186 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:31.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:31.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:31.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:31.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:31.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:31.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:31.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:31.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:27:31.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:31.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:31.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:31.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:31.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:31 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:31.877 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:31 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:31.877 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:31 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:31.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:31.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:31.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:32.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:32.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:32.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:32.157 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:32.200 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:32.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:32.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:32.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:32.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:32.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:32.653 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:32.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:32.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:32.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:32.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:32 smithi001 ceph-mon[107399]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:27:32.947 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:32 smithi001 ceph-mon[107399]: pgmap v1465: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 33683/212436 objects misplaced (15.856%); 16 MiB/s, 1 keys/s, 255 objects/s recovering 2024-09-14T04:27:32.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:32 smithi001 ceph-mon[112165]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:27:32.947 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:32 smithi001 ceph-mon[112165]: pgmap v1465: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 33683/212436 objects misplaced (15.856%); 16 MiB/s, 1 keys/s, 255 objects/s recovering 2024-09-14T04:27:32.967 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:32.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:33.007 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:32 smithi012 ceph-mon[101180]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:27:33.007 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:32 smithi012 ceph-mon[101180]: pgmap v1465: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 33683/212436 objects misplaced (15.856%); 16 MiB/s, 1 keys/s, 255 objects/s recovering 2024-09-14T04:27:33.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:33.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:33.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:33.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:33.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:33.392 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:33.494 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:33.544 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1464,"stamp":"2024-09-14T04:27:29.540587+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4906293245,"num_objects":74882,"num_object_clones":0,"num_object_copies":224646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":36477,"num_objects_unfound":0,"num_objects_dirty":74882,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":67535,"num_write_kb":2843938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5516,"num_bytes_recovered":359090440,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":162047,"ondisk_log_size":162047,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":381,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21286904,"kb_used_data":15016876,"kb_used_omap":40,"kb_used_meta":6269463,"kb_avail":634929160,"statfs":{"total":671965249536,"available":650167459840,"internally_reserved":0,"allocated":15377281024,"data_stored":15357113726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41342,"internal_metadata":6419930754},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":809205760,"num_objects":12347,"num_object_clones":0,"num_object_copies":37041,"num_objects_missing_on_primary":0,"num_objects_missing":-8,"num_objects_degraded":0,"num_objects_misplaced":27697,"num_objects_unfound":0,"num_objects_dirty":12347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15215,"num_write_kb":466592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2366,"num_bytes_recovered":154342252,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.313991"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":694,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.241555+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:27:23.241555+0000","last_peered":"2024-09-14T04:27:23.241555+0000","last_clean":"2024-09-14T04:27:23.241555+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:27:23.241555+0000","last_undegraded":"2024-09-14T04:27:23.241555+0000","last_fullsized":"2024-09-14T04:27:23.241555+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":208,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.360128+0000","last_change":"2024-09-14T04:27:20.240919+0000","last_active":"2024-09-14T04:27:25.360128+0000","last_peered":"2024-09-14T04:27:25.360128+0000","last_clean":"2024-09-14T04:27:25.360128+0000","last_became_active":"2024-09-14T04:27:20.240514+0000","last_became_peered":"2024-09-14T04:27:20.240514+0000","last_unstale":"2024-09-14T04:27:25.360128+0000","last_undegraded":"2024-09-14T04:27:25.360128+0000","last_fullsized":"2024-09-14T04:27:25.360128+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:55.818976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":20,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231566+0000","last_change":"2024-09-14T04:27:20.354646+0000","last_active":"2024-09-14T04:27:22.231566+0000","last_peered":"2024-09-14T04:27:22.231566+0000","last_clean":"2024-09-14T04:27:22.231566+0000","last_became_active":"2024-09-14T04:27:20.242671+0000","last_became_peered":"2024-09-14T04:27:20.242671+0000","last_unstale":"2024-09-14T04:27:22.231566+0000","last_undegraded":"2024-09-14T04:27:22.231566+0000","last_fullsized":"2024-09-14T04:27:22.231566+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:14.740647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":802,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.243394+0000","last_change":"2024-09-14T04:27:23.243063+0000","last_active":"2024-09-14T04:27:23.243394+0000","last_peered":"2024-09-14T04:27:23.243394+0000","last_clean":"2024-09-14T04:27:23.243394+0000","last_became_active":"2024-09-14T04:27:23.242547+0000","last_became_peered":"2024-09-14T04:27:23.242547+0000","last_unstale":"2024-09-14T04:27:23.243394+0000","last_undegraded":"2024-09-14T04:27:23.243394+0000","last_fullsized":"2024-09-14T04:27:23.243394+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:29.267328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239512+0000","last_change":"2024-09-14T04:27:20.243359+0000","last_active":"2024-09-14T04:27:23.239512+0000","last_peered":"2024-09-14T04:27:23.239512+0000","last_clean":"2024-09-14T04:27:23.239512+0000","last_became_active":"2024-09-14T04:27:20.243021+0000","last_became_peered":"2024-09-14T04:27:20.243021+0000","last_unstale":"2024-09-14T04:27:23.239512+0000","last_undegraded":"2024-09-14T04:27:23.239512+0000","last_fullsized":"2024-09-14T04:27:23.239512+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:58:10.694816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":517,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.243333+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:27:23.243333+0000","last_peered":"2024-09-14T04:27:23.243333+0000","last_clean":"2024-09-14T04:27:23.243333+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:27:23.243333+0000","last_undegraded":"2024-09-14T04:27:23.243333+0000","last_fullsized":"2024-09-14T04:27:23.243333+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":985,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359458+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:27:25.359458+0000","last_peered":"2024-09-14T04:27:25.359458+0000","last_clean":"2024-09-14T04:27:25.359458+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:27:25.359458+0000","last_undegraded":"2024-09-14T04:27:25.359458+0000","last_fullsized":"2024-09-14T04:27:25.359458+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":502,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.244093+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:27:23.244093+0000","last_peered":"2024-09-14T04:27:23.244093+0000","last_clean":"2024-09-14T04:27:23.244093+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:27:23.244093+0000","last_undegraded":"2024-09-14T04:27:23.244093+0000","last_fullsized":"2024-09-14T04:27:23.244093+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231623+0000","last_change":"2024-09-14T04:27:20.237378+0000","last_active":"2024-09-14T04:27:22.231623+0000","last_peered":"2024-09-14T04:27:22.231623+0000","last_clean":"2024-09-14T04:27:22.231623+0000","last_became_active":"2024-09-14T04:27:20.236869+0000","last_became_peered":"2024-09-14T04:27:20.236869+0000","last_unstale":"2024-09-14T04:27:22.231623+0000","last_undegraded":"2024-09-14T04:27:22.231623+0000","last_fullsized":"2024-09-14T04:27:22.231623+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:21:00.392120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":630,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242360+0000","last_change":"2024-09-14T04:27:23.241302+0000","last_active":"2024-09-14T04:27:23.242360+0000","last_peered":"2024-09-14T04:27:23.242360+0000","last_clean":"2024-09-14T04:27:23.242360+0000","last_became_active":"2024-09-14T04:27:23.240954+0000","last_became_peered":"2024-09-14T04:27:23.240954+0000","last_unstale":"2024-09-14T04:27:23.242360+0000","last_undegraded":"2024-09-14T04:27:23.242360+0000","last_fullsized":"2024-09-14T04:27:23.242360+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:35.830140+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.355565+0000","last_change":"2024-09-14T04:27:20.236945+0000","last_active":"2024-09-14T04:27:25.355565+0000","last_peered":"2024-09-14T04:27:25.355565+0000","last_clean":"2024-09-14T04:27:25.355565+0000","last_became_active":"2024-09-14T04:27:20.236600+0000","last_became_peered":"2024-09-14T04:27:20.236600+0000","last_unstale":"2024-09-14T04:27:25.355565+0000","last_undegraded":"2024-09-14T04:27:25.355565+0000","last_fullsized":"2024-09-14T04:27:25.355565+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:49.622182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":577,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.241677+0000","last_change":"2024-09-14T04:25:25.588882+0000","last_active":"2024-09-14T04:27:23.241677+0000","last_peered":"2024-09-14T04:27:23.241677+0000","last_clean":"2024-09-14T04:27:23.241677+0000","last_became_active":"2024-09-14T04:25:25.582032+0000","last_became_peered":"2024-09-14T04:25:25.582032+0000","last_unstale":"2024-09-14T04:27:23.241677+0000","last_undegraded":"2024-09-14T04:27:23.241677+0000","last_fullsized":"2024-09-14T04:27:23.241677+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:03:00.999131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1","version":"446'9360","reported_seq":9521,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:27.349555+0000","last_change":"2024-09-14T04:27:21.325545+0000","last_active":"2024-09-14T04:27:27.349555+0000","last_peered":"2024-09-14T04:27:27.349555+0000","last_clean":"2024-09-14T04:27:18.800845+0000","last_became_active":"2024-09-14T04:27:21.313983+0000","last_became_peered":"2024-09-14T04:27:21.313983+0000","last_unstale":"2024-09-14T04:27:27.349555+0000","last_undegraded":"2024-09-14T04:27:27.349555+0000","last_fullsized":"2024-09-14T04:27:27.349555+0000","mapping_epoch":442,"log_start":"409'2200","ondisk_log_start":"409'2200","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'8948","last_scrub_stamp":"2024-09-14T04:27:07.957405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.957405+0000","objects_scrubbed":2669,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146669568,"num_objects":2238,"num_object_clones":0,"num_object_copies":6714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2238,"num_objects_unfound":0,"num_objects_dirty":2238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":431,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":2238}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":713,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.241978+0000","last_change":"2024-09-14T04:27:23.241978+0000","last_active":"2024-09-14T04:27:23.241978+0000","last_peered":"2024-09-14T04:27:23.241978+0000","last_clean":"2024-09-14T04:27:23.241978+0000","last_became_active":"2024-09-14T04:27:23.241538+0000","last_became_peered":"2024-09-14T04:27:23.241538+0000","last_unstale":"2024-09-14T04:27:23.241978+0000","last_undegraded":"2024-09-14T04:27:23.241978+0000","last_fullsized":"2024-09-14T04:27:23.241978+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:14.548348+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":18,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233162+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:27:22.233162+0000","last_peered":"2024-09-14T04:27:22.233162+0000","last_clean":"2024-09-14T04:27:22.233162+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:27:22.233162+0000","last_undegraded":"2024-09-14T04:27:22.233162+0000","last_fullsized":"2024-09-14T04:27:22.233162+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":67,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.237635+0000","last_change":"2024-09-14T04:27:20.241919+0000","last_active":"2024-09-14T04:27:23.237635+0000","last_peered":"2024-09-14T04:27:23.237635+0000","last_clean":"2024-09-14T04:27:23.237635+0000","last_became_active":"2024-09-14T04:27:20.241583+0000","last_became_peered":"2024-09-14T04:27:20.241583+0000","last_unstale":"2024-09-14T04:27:23.237635+0000","last_undegraded":"2024-09-14T04:27:23.237635+0000","last_fullsized":"2024-09-14T04:27:23.237635+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:28.708616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"445'9622","reported_seq":9780,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:24.572580+0000","last_change":"2024-09-14T04:27:21.317992+0000","last_active":"2024-09-14T04:27:24.572580+0000","last_peered":"2024-09-14T04:27:24.572580+0000","last_clean":"2024-09-14T04:27:18.725450+0000","last_became_active":"2024-09-14T04:27:21.305948+0000","last_became_peered":"2024-09-14T04:27:21.305948+0000","last_unstale":"2024-09-14T04:27:24.572580+0000","last_undegraded":"2024-09-14T04:27:24.572580+0000","last_fullsized":"2024-09-14T04:27:24.572580+0000","mapping_epoch":442,"log_start":"420'3900","ondisk_log_start":"420'3900","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9193","last_scrub_stamp":"2024-09-14T04:27:06.120770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.120770+0000","objects_scrubbed":2771,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"no scrub is scheduled","scrub_duration":1072,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436736,"num_objects":2326,"num_object_clones":0,"num_object_copies":6978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2326,"num_objects_unfound":0,"num_objects_dirty":2326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":445,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[0,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2326}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1038,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.354718+0000","last_change":"2024-09-14T04:25:10.210235+0000","last_active":"2024-09-14T04:27:25.354718+0000","last_peered":"2024-09-14T04:27:25.354718+0000","last_clean":"2024-09-14T04:27:25.354718+0000","last_became_active":"2024-09-14T04:25:10.209842+0000","last_became_peered":"2024-09-14T04:25:10.209842+0000","last_unstale":"2024-09-14T04:27:25.354718+0000","last_undegraded":"2024-09-14T04:27:25.354718+0000","last_fullsized":"2024-09-14T04:27:25.354718+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:25.981272+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":18,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233864+0000","last_change":"2024-09-14T04:27:20.256429+0000","last_active":"2024-09-14T04:27:22.233864+0000","last_peered":"2024-09-14T04:27:22.233864+0000","last_clean":"2024-09-14T04:27:22.233864+0000","last_became_active":"2024-09-14T04:27:20.238968+0000","last_became_peered":"2024-09-14T04:27:20.238968+0000","last_unstale":"2024-09-14T04:27:22.233864+0000","last_undegraded":"2024-09-14T04:27:22.233864+0000","last_fullsized":"2024-09-14T04:27:22.233864+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:19.325270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":471,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231857+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:27:22.231857+0000","last_peered":"2024-09-14T04:27:22.231857+0000","last_clean":"2024-09-14T04:27:22.231857+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:27:22.231857+0000","last_undegraded":"2024-09-14T04:27:22.231857+0000","last_fullsized":"2024-09-14T04:27:22.231857+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"445'9583","reported_seq":9739,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:24.541629+0000","last_change":"2024-09-14T04:27:06.468498+0000","last_active":"2024-09-14T04:27:24.541629+0000","last_peered":"2024-09-14T04:27:24.541629+0000","last_clean":"2024-09-14T04:27:24.541629+0000","last_became_active":"2024-09-14T04:27:04.629985+0000","last_became_peered":"2024-09-14T04:27:04.629985+0000","last_unstale":"2024-09-14T04:27:24.541629+0000","last_undegraded":"2024-09-14T04:27:24.541629+0000","last_fullsized":"2024-09-14T04:27:24.541629+0000","mapping_epoch":398,"log_start":"418'3600","ondisk_log_start":"418'3600","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9192","last_scrub_stamp":"2024-09-14T04:27:06.468321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.468321+0000","objects_scrubbed":2840,"log_size":5983,"log_dups_size":0,"ondisk_log_size":5983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:23.038793+0000","scrub_duration":864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159186944,"num_objects":2429,"num_object_clones":0,"num_object_copies":7287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":411,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":468,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.240365+0000","last_change":"2024-09-14T04:24:49.565974+0000","last_active":"2024-09-14T04:27:23.240365+0000","last_peered":"2024-09-14T04:27:23.240365+0000","last_clean":"2024-09-14T04:27:23.240365+0000","last_became_active":"2024-09-14T04:24:49.565547+0000","last_became_peered":"2024-09-14T04:24:49.565547+0000","last_unstale":"2024-09-14T04:27:23.240365+0000","last_undegraded":"2024-09-14T04:27:23.240365+0000","last_fullsized":"2024-09-14T04:27:23.240365+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:04.172804+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":18,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233508+0000","last_change":"2024-09-14T04:27:20.356952+0000","last_active":"2024-09-14T04:27:22.233508+0000","last_peered":"2024-09-14T04:27:22.233508+0000","last_clean":"2024-09-14T04:27:22.233508+0000","last_became_active":"2024-09-14T04:27:20.244571+0000","last_became_peered":"2024-09-14T04:27:20.244571+0000","last_unstale":"2024-09-14T04:27:22.233508+0000","last_undegraded":"2024-09-14T04:27:22.233508+0000","last_fullsized":"2024-09-14T04:27:22.233508+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:29.308420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":491,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359221+0000","last_change":"2024-09-14T04:27:20.255490+0000","last_active":"2024-09-14T04:27:25.359221+0000","last_peered":"2024-09-14T04:27:25.359221+0000","last_clean":"2024-09-14T04:27:25.359221+0000","last_became_active":"2024-09-14T04:27:20.255200+0000","last_became_peered":"2024-09-14T04:27:20.255200+0000","last_unstale":"2024-09-14T04:27:25.359221+0000","last_undegraded":"2024-09-14T04:27:25.359221+0000","last_fullsized":"2024-09-14T04:27:25.359221+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:06.262888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"446'9586","reported_seq":15936,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.180333+0000","last_change":"2024-09-14T04:27:08.799372+0000","last_active":"2024-09-14T04:27:26.180333+0000","last_peered":"2024-09-14T04:27:26.180333+0000","last_clean":"2024-09-14T04:27:26.180333+0000","last_became_active":"2024-09-14T04:27:04.633656+0000","last_became_peered":"2024-09-14T04:27:04.633656+0000","last_unstale":"2024-09-14T04:27:26.180333+0000","last_undegraded":"2024-09-14T04:27:26.180333+0000","last_fullsized":"2024-09-14T04:27:26.180333+0000","mapping_epoch":425,"log_start":"416'3200","ondisk_log_start":"416'3200","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'9212","last_scrub_stamp":"2024-09-14T04:27:08.799175+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:08.799175+0000","objects_scrubbed":2719,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:13.865312+0000","scrub_duration":0.64896177700000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153157632,"num_objects":2337,"num_object_clones":0,"num_object_copies":7011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":382,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":451,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.253987+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:27:23.253987+0000","last_peered":"2024-09-14T04:27:23.253987+0000","last_clean":"2024-09-14T04:27:23.253987+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:27:23.253987+0000","last_undegraded":"2024-09-14T04:27:23.253987+0000","last_fullsized":"2024-09-14T04:27:23.253987+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":395,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239543+0000","last_change":"2024-09-14T04:24:07.976865+0000","last_active":"2024-09-14T04:27:23.239543+0000","last_peered":"2024-09-14T04:27:23.239543+0000","last_clean":"2024-09-14T04:27:23.239543+0000","last_became_active":"2024-09-14T04:24:07.976550+0000","last_became_peered":"2024-09-14T04:24:07.976550+0000","last_unstale":"2024-09-14T04:27:23.239543+0000","last_undegraded":"2024-09-14T04:27:23.239543+0000","last_fullsized":"2024-09-14T04:27:23.239543+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:04:02.701355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":489,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.243058+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:27:23.243058+0000","last_peered":"2024-09-14T04:27:23.243058+0000","last_clean":"2024-09-14T04:27:23.243058+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:27:23.243058+0000","last_undegraded":"2024-09-14T04:27:23.243058+0000","last_fullsized":"2024-09-14T04:27:23.243058+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"445'9719","reported_seq":9850,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.146755+0000","last_change":"2024-09-14T04:27:09.662483+0000","last_active":"2024-09-14T04:27:25.146755+0000","last_peered":"2024-09-14T04:27:25.146755+0000","last_clean":"2024-09-14T04:27:25.146755+0000","last_became_active":"2024-09-14T04:27:04.651733+0000","last_became_peered":"2024-09-14T04:27:04.651733+0000","last_unstale":"2024-09-14T04:27:25.146755+0000","last_undegraded":"2024-09-14T04:27:25.146755+0000","last_fullsized":"2024-09-14T04:27:25.146755+0000","mapping_epoch":398,"log_start":"420'4100","ondisk_log_start":"420'4100","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'9425","last_scrub_stamp":"2024-09-14T04:27:09.662329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:09.662329+0000","objects_scrubbed":2639,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:00.812272+0000","scrub_duration":0.53680546200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":298,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":780,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.258324+0000","last_change":"2024-09-14T04:27:23.258324+0000","last_active":"2024-09-14T04:27:23.258324+0000","last_peered":"2024-09-14T04:27:23.258324+0000","last_clean":"2024-09-14T04:27:23.258324+0000","last_became_active":"2024-09-14T04:27:23.257916+0000","last_became_peered":"2024-09-14T04:27:23.257916+0000","last_unstale":"2024-09-14T04:27:23.258324+0000","last_undegraded":"2024-09-14T04:27:23.258324+0000","last_fullsized":"2024-09-14T04:27:23.258324+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:29:08.763999+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":493,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242751+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:27:23.242751+0000","last_peered":"2024-09-14T04:27:23.242751+0000","last_clean":"2024-09-14T04:27:23.242751+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:27:23.242751+0000","last_undegraded":"2024-09-14T04:27:23.242751+0000","last_fullsized":"2024-09-14T04:27:23.242751+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":446,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233908+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:27:22.233908+0000","last_peered":"2024-09-14T04:27:22.233908+0000","last_clean":"2024-09-14T04:27:22.233908+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:27:22.233908+0000","last_undegraded":"2024-09-14T04:27:22.233908+0000","last_fullsized":"2024-09-14T04:27:22.233908+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"446'9841","reported_seq":9998,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:27.363077+0000","last_change":"2024-09-14T04:27:07.089728+0000","last_active":"2024-09-14T04:27:27.363077+0000","last_peered":"2024-09-14T04:27:27.363077+0000","last_clean":"2024-09-14T04:27:27.363077+0000","last_became_active":"2024-09-14T04:27:04.639444+0000","last_became_peered":"2024-09-14T04:27:04.639444+0000","last_unstale":"2024-09-14T04:27:27.363077+0000","last_undegraded":"2024-09-14T04:27:27.363077+0000","last_fullsized":"2024-09-14T04:27:27.363077+0000","mapping_epoch":398,"log_start":"412'2700","ondisk_log_start":"412'2700","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9334","last_scrub_stamp":"2024-09-14T04:27:07.089071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.089071+0000","objects_scrubbed":2819,"log_size":7141,"log_dups_size":0,"ondisk_log_size":7141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:30.483229+0000","scrub_duration":1039,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":523,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":680,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242265+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:27:23.242265+0000","last_peered":"2024-09-14T04:27:23.242265+0000","last_clean":"2024-09-14T04:27:23.242265+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:27:23.242265+0000","last_undegraded":"2024-09-14T04:27:23.242265+0000","last_fullsized":"2024-09-14T04:27:23.242265+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":571,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.243545+0000","last_change":"2024-09-14T04:23:27.570715+0000","last_active":"2024-09-14T04:27:23.243545+0000","last_peered":"2024-09-14T04:27:23.243545+0000","last_clean":"2024-09-14T04:27:23.243545+0000","last_became_active":"2024-09-14T04:23:27.570413+0000","last_became_peered":"2024-09-14T04:23:27.570413+0000","last_unstale":"2024-09-14T04:27:23.243545+0000","last_undegraded":"2024-09-14T04:27:23.243545+0000","last_fullsized":"2024-09-14T04:27:23.243545+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:20.405121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.240778+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:27:23.240778+0000","last_peered":"2024-09-14T04:27:23.240778+0000","last_clean":"2024-09-14T04:27:23.240778+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:27:23.240778+0000","last_undegraded":"2024-09-14T04:27:23.240778+0000","last_fullsized":"2024-09-14T04:27:23.240778+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"445'9659","reported_seq":14379,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:24.578756+0000","last_change":"2024-09-14T04:27:21.330925+0000","last_active":"2024-09-14T04:27:24.578756+0000","last_peered":"2024-09-14T04:27:24.578756+0000","last_clean":"2024-09-14T04:27:18.772376+0000","last_became_active":"2024-09-14T04:27:21.316475+0000","last_became_peered":"2024-09-14T04:27:21.316475+0000","last_unstale":"2024-09-14T04:27:24.578756+0000","last_undegraded":"2024-09-14T04:27:24.578756+0000","last_fullsized":"2024-09-14T04:27:24.578756+0000","mapping_epoch":442,"log_start":"420'4100","ondisk_log_start":"420'4100","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9272","last_scrub_stamp":"2024-09-14T04:27:06.713334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.713334+0000","objects_scrubbed":2843,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.58335741600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159776768,"num_objects":2438,"num_object_clones":0,"num_object_copies":7314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2438,"num_objects_unfound":0,"num_objects_dirty":2438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":405,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[0,4,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":2438}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":965,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359134+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:27:25.359134+0000","last_peered":"2024-09-14T04:27:25.359134+0000","last_clean":"2024-09-14T04:27:25.359134+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:27:25.359134+0000","last_undegraded":"2024-09-14T04:27:25.359134+0000","last_fullsized":"2024-09-14T04:27:25.359134+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":64,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231587+0000","last_change":"2024-09-14T04:27:20.244483+0000","last_active":"2024-09-14T04:27:22.231587+0000","last_peered":"2024-09-14T04:27:22.231587+0000","last_clean":"2024-09-14T04:27:22.231587+0000","last_became_active":"2024-09-14T04:27:20.244182+0000","last_became_peered":"2024-09-14T04:27:20.244182+0000","last_unstale":"2024-09-14T04:27:22.231587+0000","last_undegraded":"2024-09-14T04:27:22.231587+0000","last_fullsized":"2024-09-14T04:27:22.231587+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:36.658365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":452,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.236092+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:27:22.236092+0000","last_peered":"2024-09-14T04:27:22.236092+0000","last_clean":"2024-09-14T04:27:22.236092+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:27:22.236092+0000","last_undegraded":"2024-09-14T04:27:22.236092+0000","last_fullsized":"2024-09-14T04:27:22.236092+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"445'9545","reported_seq":9680,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.130843+0000","last_change":"2024-09-14T04:27:05.659073+0000","last_active":"2024-09-14T04:27:25.130843+0000","last_peered":"2024-09-14T04:27:25.130843+0000","last_clean":"2024-09-14T04:27:25.130843+0000","last_became_active":"2024-09-14T04:27:04.643791+0000","last_became_peered":"2024-09-14T04:27:04.643791+0000","last_unstale":"2024-09-14T04:27:25.130843+0000","last_undegraded":"2024-09-14T04:27:25.130843+0000","last_fullsized":"2024-09-14T04:27:25.130843+0000","mapping_epoch":398,"log_start":"420'3900","ondisk_log_start":"420'3900","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9109","last_scrub_stamp":"2024-09-14T04:27:05.658918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:05.658918+0000","objects_scrubbed":2855,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:50.057418+0000","scrub_duration":0.52478999100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":449,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":651,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.238399+0000","last_change":"2024-09-14T04:24:45.568702+0000","last_active":"2024-09-14T04:27:23.238399+0000","last_peered":"2024-09-14T04:27:23.238399+0000","last_clean":"2024-09-14T04:27:23.238399+0000","last_became_active":"2024-09-14T04:24:07.986545+0000","last_became_peered":"2024-09-14T04:24:07.986545+0000","last_unstale":"2024-09-14T04:27:23.238399+0000","last_undegraded":"2024-09-14T04:27:23.238399+0000","last_fullsized":"2024-09-14T04:27:23.238399+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:36:33.664214+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":83,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.353547+0000","last_change":"2024-09-14T04:27:20.236754+0000","last_active":"2024-09-14T04:27:25.353547+0000","last_peered":"2024-09-14T04:27:25.353547+0000","last_clean":"2024-09-14T04:27:25.353547+0000","last_became_active":"2024-09-14T04:27:20.236432+0000","last_became_peered":"2024-09-14T04:27:20.236432+0000","last_unstale":"2024-09-14T04:27:25.353547+0000","last_undegraded":"2024-09-14T04:27:25.353547+0000","last_fullsized":"2024-09-14T04:27:25.353547+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:43.774063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":735,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.360214+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:27:25.360214+0000","last_peered":"2024-09-14T04:27:25.360214+0000","last_clean":"2024-09-14T04:27:25.360214+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:27:25.360214+0000","last_undegraded":"2024-09-14T04:27:25.360214+0000","last_fullsized":"2024-09-14T04:27:25.360214+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"446'10309","reported_seq":10424,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.059706+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:27:26.059706+0000","last_peered":"2024-09-14T04:27:26.059706+0000","last_clean":"2024-09-14T04:27:26.059706+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:27:26.059706+0000","last_undegraded":"2024-09-14T04:27:26.059706+0000","last_fullsized":"2024-09-14T04:27:26.059706+0000","mapping_epoch":398,"log_start":"423'5183","ondisk_log_start":"423'5183","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:38:53.553101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310575104,"num_objects":4739,"num_object_clones":0,"num_object_copies":14217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10309,"num_write_kb":481536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":835,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242726+0000","last_change":"2024-09-14T04:27:23.242726+0000","last_active":"2024-09-14T04:27:23.242726+0000","last_peered":"2024-09-14T04:27:23.242726+0000","last_clean":"2024-09-14T04:27:23.242726+0000","last_became_active":"2024-09-14T04:27:23.242377+0000","last_became_peered":"2024-09-14T04:27:23.242377+0000","last_unstale":"2024-09-14T04:27:23.242726+0000","last_undegraded":"2024-09-14T04:27:23.242726+0000","last_fullsized":"2024-09-14T04:27:23.242726+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:53.189357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":398,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.238310+0000","last_change":"2024-09-14T04:27:20.357956+0000","last_active":"2024-09-14T04:27:23.238310+0000","last_peered":"2024-09-14T04:27:23.238310+0000","last_clean":"2024-09-14T04:27:23.238310+0000","last_became_active":"2024-09-14T04:27:20.241613+0000","last_became_peered":"2024-09-14T04:27:20.241613+0000","last_unstale":"2024-09-14T04:27:23.238310+0000","last_undegraded":"2024-09-14T04:27:23.238310+0000","last_fullsized":"2024-09-14T04:27:23.238310+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:06.823240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":60,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.358567+0000","last_change":"2024-09-14T04:25:52.594667+0000","last_active":"2024-09-14T04:27:25.358567+0000","last_peered":"2024-09-14T04:27:25.358567+0000","last_clean":"2024-09-14T04:27:25.358567+0000","last_became_active":"2024-09-14T04:25:52.594400+0000","last_became_peered":"2024-09-14T04:25:52.594400+0000","last_unstale":"2024-09-14T04:27:25.358567+0000","last_undegraded":"2024-09-14T04:27:25.358567+0000","last_fullsized":"2024-09-14T04:27:25.358567+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:12.710417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.235535+0000","last_change":"2024-09-14T04:27:20.247824+0000","last_active":"2024-09-14T04:27:22.235535+0000","last_peered":"2024-09-14T04:27:22.235535+0000","last_clean":"2024-09-14T04:27:22.235535+0000","last_became_active":"2024-09-14T04:27:20.247205+0000","last_became_peered":"2024-09-14T04:27:20.247205+0000","last_unstale":"2024-09-14T04:27:22.235535+0000","last_undegraded":"2024-09-14T04:27:22.235535+0000","last_fullsized":"2024-09-14T04:27:22.235535+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:09.902975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"446'10111","reported_seq":10235,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:27.384753+0000","last_change":"2024-09-14T04:27:21.395284+0000","last_active":"2024-09-14T04:27:27.384753+0000","last_peered":"2024-09-14T04:27:27.384753+0000","last_clean":"2024-09-14T04:27:18.861577+0000","last_became_active":"2024-09-14T04:27:21.368022+0000","last_became_peered":"2024-09-14T04:27:21.368022+0000","last_unstale":"2024-09-14T04:27:27.384753+0000","last_undegraded":"2024-09-14T04:27:27.384753+0000","last_fullsized":"2024-09-14T04:27:27.384753+0000","mapping_epoch":442,"log_start":"420'3750","ondisk_log_start":"420'3750","created":398,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6361,"log_dups_size":0,"ondisk_log_size":6361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875776,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4591,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10111,"num_write_kb":470464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":4591}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":957,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359243+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:27:25.359243+0000","last_peered":"2024-09-14T04:27:25.359243+0000","last_clean":"2024-09-14T04:27:25.359243+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:27:25.359243+0000","last_undegraded":"2024-09-14T04:27:25.359243+0000","last_fullsized":"2024-09-14T04:27:25.359243+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.230867+0000","last_change":"2024-09-14T04:27:20.248193+0000","last_active":"2024-09-14T04:27:22.230867+0000","last_peered":"2024-09-14T04:27:22.230867+0000","last_clean":"2024-09-14T04:27:22.230867+0000","last_became_active":"2024-09-14T04:27:20.247785+0000","last_became_peered":"2024-09-14T04:27:20.247785+0000","last_unstale":"2024-09-14T04:27:22.230867+0000","last_undegraded":"2024-09-14T04:27:22.230867+0000","last_fullsized":"2024-09-14T04:27:22.230867+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:44.759346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":739,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.241440+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:27:23.241440+0000","last_peered":"2024-09-14T04:27:23.241440+0000","last_clean":"2024-09-14T04:27:23.241440+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:27:23.241440+0000","last_undegraded":"2024-09-14T04:27:23.241440+0000","last_fullsized":"2024-09-14T04:27:23.241440+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":500,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.244200+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:27:23.244200+0000","last_peered":"2024-09-14T04:27:23.244200+0000","last_clean":"2024-09-14T04:27:23.244200+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:27:23.244200+0000","last_undegraded":"2024-09-14T04:27:23.244200+0000","last_fullsized":"2024-09-14T04:27:23.244200+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"445'9880","reported_seq":9945,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:24.555037+0000","last_change":"2024-09-14T04:27:21.366294+0000","last_active":"2024-09-14T04:27:24.555037+0000","last_peered":"2024-09-14T04:27:24.555037+0000","last_clean":"2024-09-14T04:27:18.812457+0000","last_became_active":"2024-09-14T04:27:21.338728+0000","last_became_peered":"2024-09-14T04:27:21.338728+0000","last_unstale":"2024-09-14T04:27:24.555037+0000","last_undegraded":"2024-09-14T04:27:24.555037+0000","last_fullsized":"2024-09-14T04:27:24.555037+0000","mapping_epoch":442,"log_start":"423'4730","ondisk_log_start":"423'4730","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4714,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":4714}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":710,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.245485+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:27:23.245485+0000","last_peered":"2024-09-14T04:27:23.245485+0000","last_clean":"2024-09-14T04:27:23.245485+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:27:23.245485+0000","last_undegraded":"2024-09-14T04:27:23.245485+0000","last_fullsized":"2024-09-14T04:27:23.245485+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":708,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.361236+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:27:25.361236+0000","last_peered":"2024-09-14T04:27:25.361236+0000","last_clean":"2024-09-14T04:27:25.361236+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:27:25.361236+0000","last_undegraded":"2024-09-14T04:27:25.361236+0000","last_fullsized":"2024-09-14T04:27:25.361236+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":739,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359621+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:27:25.359621+0000","last_peered":"2024-09-14T04:27:25.359621+0000","last_clean":"2024-09-14T04:27:25.359621+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:27:25.359621+0000","last_undegraded":"2024-09-14T04:27:25.359621+0000","last_fullsized":"2024-09-14T04:27:25.359621+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":207,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.360009+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:27:25.360009+0000","last_peered":"2024-09-14T04:27:25.360009+0000","last_clean":"2024-09-14T04:27:25.360009+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:27:25.360009+0000","last_undegraded":"2024-09-14T04:27:25.360009+0000","last_fullsized":"2024-09-14T04:27:25.360009+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"445'10110","reported_seq":10178,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:25.171057+0000","last_change":"2024-09-14T04:27:21.341809+0000","last_active":"2024-09-14T04:27:25.171057+0000","last_peered":"2024-09-14T04:27:25.171057+0000","last_clean":"2024-09-14T04:27:18.848403+0000","last_became_active":"2024-09-14T04:27:21.321221+0000","last_became_peered":"2024-09-14T04:27:21.321221+0000","last_unstale":"2024-09-14T04:27:25.171057+0000","last_undegraded":"2024-09-14T04:27:25.171057+0000","last_fullsized":"2024-09-14T04:27:25.171057+0000","mapping_epoch":442,"log_start":"423'5245","ondisk_log_start":"423'5245","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":4865,"log_dups_size":0,"ondisk_log_size":4865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:40:51.836245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316604440,"num_objects":4832,"num_object_clones":0,"num_object_copies":14496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4832,"num_objects_unfound":0,"num_objects_dirty":4832,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10110,"num_write_kb":478081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":4832}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":674,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.237610+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:27:23.237610+0000","last_peered":"2024-09-14T04:27:23.237610+0000","last_clean":"2024-09-14T04:27:23.237610+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:27:23.237610+0000","last_undegraded":"2024-09-14T04:27:23.237610+0000","last_fullsized":"2024-09-14T04:27:23.237610+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":382,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.238305+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:27:23.238305+0000","last_peered":"2024-09-14T04:27:23.238305+0000","last_clean":"2024-09-14T04:27:23.238305+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:27:23.238305+0000","last_undegraded":"2024-09-14T04:27:23.238305+0000","last_fullsized":"2024-09-14T04:27:23.238305+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":464,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234691+0000","last_change":"2024-09-14T04:21:43.502521+0000","last_active":"2024-09-14T04:27:22.234691+0000","last_peered":"2024-09-14T04:27:22.234691+0000","last_clean":"2024-09-14T04:27:22.234691+0000","last_became_active":"2024-09-14T04:21:43.502220+0000","last_became_peered":"2024-09-14T04:21:43.502220+0000","last_unstale":"2024-09-14T04:27:22.234691+0000","last_undegraded":"2024-09-14T04:27:22.234691+0000","last_fullsized":"2024-09-14T04:27:22.234691+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:18.713496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":402,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239616+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:27:23.239616+0000","last_peered":"2024-09-14T04:27:23.239616+0000","last_clean":"2024-09-14T04:27:23.239616+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:27:23.239616+0000","last_undegraded":"2024-09-14T04:27:23.239616+0000","last_fullsized":"2024-09-14T04:27:23.239616+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"446'10226","reported_seq":10284,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.206635+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:27:26.206635+0000","last_peered":"2024-09-14T04:27:26.206635+0000","last_clean":"2024-09-14T04:27:26.206635+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:27:26.206635+0000","last_undegraded":"2024-09-14T04:27:26.206635+0000","last_fullsized":"2024-09-14T04:27:26.206635+0000","mapping_epoch":398,"log_start":"423'5084","ondisk_log_start":"423'5084","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5142,"log_dups_size":0,"ondisk_log_size":5142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311296000,"num_objects":4750,"num_object_clones":0,"num_object_copies":14250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10226,"num_write_kb":479232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":738,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.355454+0000","last_change":"2024-09-14T04:27:23.264670+0000","last_active":"2024-09-14T04:27:25.355454+0000","last_peered":"2024-09-14T04:27:25.355454+0000","last_clean":"2024-09-14T04:27:25.355454+0000","last_became_active":"2024-09-14T04:27:23.264241+0000","last_became_peered":"2024-09-14T04:27:23.264241+0000","last_unstale":"2024-09-14T04:27:25.355454+0000","last_undegraded":"2024-09-14T04:27:25.355454+0000","last_fullsized":"2024-09-14T04:27:25.355454+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:27.406676+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":411,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239295+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:27:23.239295+0000","last_peered":"2024-09-14T04:27:23.239295+0000","last_clean":"2024-09-14T04:27:23.239295+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:27:23.239295+0000","last_undegraded":"2024-09-14T04:27:23.239295+0000","last_fullsized":"2024-09-14T04:27:23.239295+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":804,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.360395+0000","last_change":"2024-09-14T04:27:20.252461+0000","last_active":"2024-09-14T04:27:25.360395+0000","last_peered":"2024-09-14T04:27:25.360395+0000","last_clean":"2024-09-14T04:27:25.360395+0000","last_became_active":"2024-09-14T04:27:20.252036+0000","last_became_peered":"2024-09-14T04:27:20.252036+0000","last_unstale":"2024-09-14T04:27:25.360395+0000","last_undegraded":"2024-09-14T04:27:25.360395+0000","last_fullsized":"2024-09-14T04:27:25.360395+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:32:51.681202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":222,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359126+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:27:25.359126+0000","last_peered":"2024-09-14T04:27:25.359126+0000","last_clean":"2024-09-14T04:27:25.359126+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:27:25.359126+0000","last_undegraded":"2024-09-14T04:27:25.359126+0000","last_fullsized":"2024-09-14T04:27:25.359126+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"446'10048","reported_seq":10163,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:27.353020+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:27:27.353020+0000","last_peered":"2024-09-14T04:27:27.353020+0000","last_clean":"2024-09-14T04:27:27.353020+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:27:27.353020+0000","last_undegraded":"2024-09-14T04:27:27.353020+0000","last_fullsized":"2024-09-14T04:27:27.353020+0000","mapping_epoch":398,"log_start":"420'3794","ondisk_log_start":"420'3794","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6254,"log_dups_size":0,"ondisk_log_size":6254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:33.580136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299106304,"num_objects":4564,"num_object_clones":0,"num_object_copies":13692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10048,"num_write_kb":467584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":744,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242287+0000","last_change":"2024-09-14T04:22:17.613647+0000","last_active":"2024-09-14T04:27:23.242287+0000","last_peered":"2024-09-14T04:27:23.242287+0000","last_clean":"2024-09-14T04:27:23.242287+0000","last_became_active":"2024-09-14T04:21:46.500714+0000","last_became_peered":"2024-09-14T04:21:46.500714+0000","last_unstale":"2024-09-14T04:27:23.242287+0000","last_undegraded":"2024-09-14T04:27:23.242287+0000","last_fullsized":"2024-09-14T04:27:23.242287+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:01.335413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":743,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.360314+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:27:25.360314+0000","last_peered":"2024-09-14T04:27:25.360314+0000","last_clean":"2024-09-14T04:27:25.360314+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:27:25.360314+0000","last_undegraded":"2024-09-14T04:27:25.360314+0000","last_fullsized":"2024-09-14T04:27:25.360314+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.232709+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:27:22.232709+0000","last_peered":"2024-09-14T04:27:22.232709+0000","last_clean":"2024-09-14T04:27:22.232709+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:27:22.232709+0000","last_undegraded":"2024-09-14T04:27:22.232709+0000","last_fullsized":"2024-09-14T04:27:22.232709+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":513,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242966+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:27:23.242966+0000","last_peered":"2024-09-14T04:27:23.242966+0000","last_clean":"2024-09-14T04:27:23.242966+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:27:23.242966+0000","last_undegraded":"2024-09-14T04:27:23.242966+0000","last_fullsized":"2024-09-14T04:27:23.242966+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"446'12033","reported_seq":12317,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.186269+0000","last_change":"2024-09-14T04:27:16.935967+0000","last_active":"2024-09-14T04:27:26.186269+0000","last_peered":"2024-09-14T04:27:26.186269+0000","last_clean":"2024-09-14T04:27:26.186269+0000","last_became_active":"2024-09-14T04:27:04.683999+0000","last_became_peered":"2024-09-14T04:27:04.683999+0000","last_unstale":"2024-09-14T04:27:26.186269+0000","last_undegraded":"2024-09-14T04:27:26.186269+0000","last_fullsized":"2024-09-14T04:27:26.186269+0000","mapping_epoch":398,"log_start":"423'4974","ondisk_log_start":"423'4974","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"438'11855","last_scrub_stamp":"2024-09-14T04:27:16.935785+0000","last_deep_scrub":"438'11855","last_deep_scrub_stamp":"2024-09-14T04:27:16.935785+0000","last_clean_scrub_stamp":"2024-09-14T04:27:16.935785+0000","objects_scrubbed":2574,"log_size":7059,"log_dups_size":0,"ondisk_log_size":7059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:33.473629+0000","scrub_duration":7.8014888280000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153026560,"num_objects":2335,"num_object_clones":0,"num_object_copies":7005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":239,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":751,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234984+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:27:22.234984+0000","last_peered":"2024-09-14T04:27:22.234984+0000","last_clean":"2024-09-14T04:27:22.234984+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:27:22.234984+0000","last_undegraded":"2024-09-14T04:27:22.234984+0000","last_fullsized":"2024-09-14T04:27:22.234984+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":18,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233457+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:27:22.233457+0000","last_peered":"2024-09-14T04:27:22.233457+0000","last_clean":"2024-09-14T04:27:22.233457+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:27:22.233457+0000","last_undegraded":"2024-09-14T04:27:22.233457+0000","last_fullsized":"2024-09-14T04:27:22.233457+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":728,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359835+0000","last_change":"2024-09-14T04:21:43.501127+0000","last_active":"2024-09-14T04:27:25.359835+0000","last_peered":"2024-09-14T04:27:25.359835+0000","last_clean":"2024-09-14T04:27:25.359835+0000","last_became_active":"2024-09-14T04:21:43.500770+0000","last_became_peered":"2024-09-14T04:21:43.500770+0000","last_unstale":"2024-09-14T04:27:25.359835+0000","last_undegraded":"2024-09-14T04:27:25.359835+0000","last_fullsized":"2024-09-14T04:27:25.359835+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:50.030381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234335+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:27:22.234335+0000","last_peered":"2024-09-14T04:27:22.234335+0000","last_clean":"2024-09-14T04:27:22.234335+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:27:22.234335+0000","last_undegraded":"2024-09-14T04:27:22.234335+0000","last_fullsized":"2024-09-14T04:27:22.234335+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"446'9483","reported_seq":9641,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:27.359425+0000","last_change":"2024-09-14T04:27:06.233426+0000","last_active":"2024-09-14T04:27:27.359425+0000","last_peered":"2024-09-14T04:27:27.359425+0000","last_clean":"2024-09-14T04:27:27.359425+0000","last_became_active":"2024-09-14T04:27:04.639182+0000","last_became_peered":"2024-09-14T04:27:04.639182+0000","last_unstale":"2024-09-14T04:27:27.359425+0000","last_undegraded":"2024-09-14T04:27:27.359425+0000","last_fullsized":"2024-09-14T04:27:27.359425+0000","mapping_epoch":398,"log_start":"410'2400","ondisk_log_start":"410'2400","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9002","last_scrub_stamp":"2024-09-14T04:27:06.233137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.233137+0000","objects_scrubbed":2784,"log_size":7083,"log_dups_size":0,"ondisk_log_size":7083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:05:34.359395+0000","scrub_duration":1174,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149028864,"num_objects":2274,"num_object_clones":0,"num_object_copies":6822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":510,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1028,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.354619+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:27:25.354619+0000","last_peered":"2024-09-14T04:27:25.354619+0000","last_clean":"2024-09-14T04:27:25.354619+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:27:25.354619+0000","last_undegraded":"2024-09-14T04:27:25.354619+0000","last_fullsized":"2024-09-14T04:27:25.354619+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":504,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242469+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:27:23.242469+0000","last_peered":"2024-09-14T04:27:23.242469+0000","last_clean":"2024-09-14T04:27:23.242469+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:27:23.242469+0000","last_undegraded":"2024-09-14T04:27:23.242469+0000","last_fullsized":"2024-09-14T04:27:23.242469+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.233288+0000","last_change":"2024-09-14T04:27:20.243542+0000","last_active":"2024-09-14T04:27:22.233288+0000","last_peered":"2024-09-14T04:27:22.233288+0000","last_clean":"2024-09-14T04:27:22.233288+0000","last_became_active":"2024-09-14T04:27:20.243250+0000","last_became_peered":"2024-09-14T04:27:20.243250+0000","last_unstale":"2024-09-14T04:27:22.233288+0000","last_undegraded":"2024-09-14T04:27:22.233288+0000","last_fullsized":"2024-09-14T04:27:22.233288+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:26:11.257845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":430,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231929+0000","last_change":"2024-09-14T04:24:07.977610+0000","last_active":"2024-09-14T04:27:22.231929+0000","last_peered":"2024-09-14T04:27:22.231929+0000","last_clean":"2024-09-14T04:27:22.231929+0000","last_became_active":"2024-09-14T04:24:07.977263+0000","last_became_peered":"2024-09-14T04:24:07.977263+0000","last_unstale":"2024-09-14T04:27:22.231929+0000","last_undegraded":"2024-09-14T04:27:22.231929+0000","last_fullsized":"2024-09-14T04:27:22.231929+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:49.665659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":467,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234488+0000","last_change":"2024-09-14T04:24:07.986729+0000","last_active":"2024-09-14T04:27:22.234488+0000","last_peered":"2024-09-14T04:27:22.234488+0000","last_clean":"2024-09-14T04:27:22.234488+0000","last_became_active":"2024-09-14T04:24:07.986355+0000","last_became_peered":"2024-09-14T04:24:07.986355+0000","last_unstale":"2024-09-14T04:27:22.234488+0000","last_undegraded":"2024-09-14T04:27:22.234488+0000","last_fullsized":"2024-09-14T04:27:22.234488+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:33.623702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":485,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234002+0000","last_change":"2024-09-14T04:27:20.260549+0000","last_active":"2024-09-14T04:27:22.234002+0000","last_peered":"2024-09-14T04:27:22.234002+0000","last_clean":"2024-09-14T04:27:22.234002+0000","last_became_active":"2024-09-14T04:27:20.240697+0000","last_became_peered":"2024-09-14T04:27:20.240697+0000","last_unstale":"2024-09-14T04:27:22.234002+0000","last_undegraded":"2024-09-14T04:27:22.234002+0000","last_fullsized":"2024-09-14T04:27:22.234002+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:50.916841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14268,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":193,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.358900+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:27:25.358900+0000","last_peered":"2024-09-14T04:27:25.358900+0000","last_clean":"2024-09-14T04:27:25.358900+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:27:25.358900+0000","last_undegraded":"2024-09-14T04:27:25.358900+0000","last_fullsized":"2024-09-14T04:27:25.358900+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":20,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.232630+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:27:22.232630+0000","last_peered":"2024-09-14T04:27:22.232630+0000","last_clean":"2024-09-14T04:27:22.232630+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:27:22.232630+0000","last_undegraded":"2024-09-14T04:27:22.232630+0000","last_fullsized":"2024-09-14T04:27:22.232630+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.18","version":"446'9476","reported_seq":9630,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:27.342755+0000","last_change":"2024-09-14T04:27:09.911394+0000","last_active":"2024-09-14T04:27:27.342755+0000","last_peered":"2024-09-14T04:27:27.342755+0000","last_clean":"2024-09-14T04:27:27.342755+0000","last_became_active":"2024-09-14T04:27:04.687912+0000","last_became_peered":"2024-09-14T04:27:04.687912+0000","last_unstale":"2024-09-14T04:27:27.342755+0000","last_undegraded":"2024-09-14T04:27:27.342755+0000","last_fullsized":"2024-09-14T04:27:27.342755+0000","mapping_epoch":430,"log_start":"410'2400","ondisk_log_start":"410'2400","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9141","last_scrub_stamp":"2024-09-14T04:27:09.911112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:09.911112+0000","objects_scrubbed":2594,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:34.459301+0000","scrub_duration":784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147193856,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":348,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":963,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.364711+0000","last_change":"2024-09-14T04:27:23.254868+0000","last_active":"2024-09-14T04:27:25.364711+0000","last_peered":"2024-09-14T04:27:25.364711+0000","last_clean":"2024-09-14T04:27:25.364711+0000","last_became_active":"2024-09-14T04:27:23.254269+0000","last_became_peered":"2024-09-14T04:27:23.254269+0000","last_unstale":"2024-09-14T04:27:25.364711+0000","last_undegraded":"2024-09-14T04:27:25.364711+0000","last_fullsized":"2024-09-14T04:27:25.364711+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:06:43.559813+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":347,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.241903+0000","last_change":"2024-09-14T04:27:20.244084+0000","last_active":"2024-09-14T04:27:23.241903+0000","last_peered":"2024-09-14T04:27:23.241903+0000","last_clean":"2024-09-14T04:27:23.241903+0000","last_became_active":"2024-09-14T04:27:20.243703+0000","last_became_peered":"2024-09-14T04:27:20.243703+0000","last_unstale":"2024-09-14T04:27:23.241903+0000","last_undegraded":"2024-09-14T04:27:23.241903+0000","last_fullsized":"2024-09-14T04:27:23.241903+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:34.342306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231709+0000","last_change":"2024-09-14T04:27:20.243792+0000","last_active":"2024-09-14T04:27:22.231709+0000","last_peered":"2024-09-14T04:27:22.231709+0000","last_clean":"2024-09-14T04:27:22.231709+0000","last_became_active":"2024-09-14T04:27:20.243463+0000","last_became_peered":"2024-09-14T04:27:20.243463+0000","last_unstale":"2024-09-14T04:27:22.231709+0000","last_undegraded":"2024-09-14T04:27:22.231709+0000","last_fullsized":"2024-09-14T04:27:22.231709+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:20.998497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.19","version":"446'9427","reported_seq":9554,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.208445+0000","last_change":"2024-09-14T04:27:22.628893+0000","last_active":"2024-09-14T04:27:26.208445+0000","last_peered":"2024-09-14T04:27:26.208445+0000","last_clean":"2024-09-14T04:27:26.208445+0000","last_became_active":"2024-09-14T04:27:04.697978+0000","last_became_peered":"2024-09-14T04:27:04.697978+0000","last_unstale":"2024-09-14T04:27:26.208445+0000","last_undegraded":"2024-09-14T04:27:26.208445+0000","last_fullsized":"2024-09-14T04:27:26.208445+0000","mapping_epoch":430,"log_start":"416'3100","ondisk_log_start":"416'3100","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"444'9315","last_scrub_stamp":"2024-09-14T04:27:22.628541+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:22.628541+0000","objects_scrubbed":2476,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:05.021923+0000","scrub_duration":0.47792231099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154206208,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":123,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":499,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.244804+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:27:23.244804+0000","last_peered":"2024-09-14T04:27:23.244804+0000","last_clean":"2024-09-14T04:27:23.244804+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:27:23.244804+0000","last_undegraded":"2024-09-14T04:27:23.244804+0000","last_fullsized":"2024-09-14T04:27:23.244804+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":605,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.353844+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:27:25.353844+0000","last_peered":"2024-09-14T04:27:25.353844+0000","last_clean":"2024-09-14T04:27:25.353844+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:27:25.353844+0000","last_undegraded":"2024-09-14T04:27:25.353844+0000","last_fullsized":"2024-09-14T04:27:25.353844+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.16","version":"445'9543","reported_seq":9670,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.168319+0000","last_change":"2024-09-14T04:27:24.501257+0000","last_active":"2024-09-14T04:27:25.168319+0000","last_peered":"2024-09-14T04:27:25.168319+0000","last_clean":"2024-09-14T04:27:25.168319+0000","last_became_active":"2024-09-14T04:27:04.657767+0000","last_became_peered":"2024-09-14T04:27:04.657767+0000","last_unstale":"2024-09-14T04:27:25.168319+0000","last_undegraded":"2024-09-14T04:27:25.168319+0000","last_fullsized":"2024-09-14T04:27:25.168319+0000","mapping_epoch":430,"log_start":"420'3900","ondisk_log_start":"420'3900","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"445'9522","last_scrub_stamp":"2024-09-14T04:27:24.500816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:24.500816+0000","objects_scrubbed":2391,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:19:55.145284+0000","scrub_duration":0.56814457399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154861568,"num_objects":2363,"num_object_clones":0,"num_object_copies":7089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":28,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":160,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.240281+0000","last_change":"2024-09-14T04:24:07.981020+0000","last_active":"2024-09-14T04:27:23.240281+0000","last_peered":"2024-09-14T04:27:23.240281+0000","last_clean":"2024-09-14T04:27:23.240281+0000","last_became_active":"2024-09-14T04:24:07.980782+0000","last_became_peered":"2024-09-14T04:24:07.980782+0000","last_unstale":"2024-09-14T04:27:23.240281+0000","last_undegraded":"2024-09-14T04:27:23.240281+0000","last_fullsized":"2024-09-14T04:27:23.240281+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:13.659164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.237479+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:27:23.237479+0000","last_peered":"2024-09-14T04:27:23.237479+0000","last_clean":"2024-09-14T04:27:23.237479+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:27:23.237479+0000","last_undegraded":"2024-09-14T04:27:23.237479+0000","last_fullsized":"2024-09-14T04:27:23.237479+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.17","version":"445'9663","reported_seq":14377,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:24.578574+0000","last_change":"2024-09-14T04:27:21.330920+0000","last_active":"2024-09-14T04:27:24.578574+0000","last_peered":"2024-09-14T04:27:24.578574+0000","last_clean":"2024-09-14T04:27:18.806026+0000","last_became_active":"2024-09-14T04:27:21.316296+0000","last_became_peered":"2024-09-14T04:27:21.316296+0000","last_unstale":"2024-09-14T04:27:24.578574+0000","last_undegraded":"2024-09-14T04:27:24.578574+0000","last_fullsized":"2024-09-14T04:27:24.578574+0000","mapping_epoch":442,"log_start":"420'4100","ondisk_log_start":"420'4100","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"433'9489","last_scrub_stamp":"2024-09-14T04:27:12.513993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:12.513993+0000","objects_scrubbed":2501,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.43811768600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174592,"num_objects":2322,"num_object_clones":0,"num_object_copies":6966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2322,"num_objects_unfound":0,"num_objects_dirty":2322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":179,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[0,4,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":2322}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":523,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242378+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:27:23.242378+0000","last_peered":"2024-09-14T04:27:23.242378+0000","last_clean":"2024-09-14T04:27:23.242378+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:27:23.242378+0000","last_undegraded":"2024-09-14T04:27:23.242378+0000","last_fullsized":"2024-09-14T04:27:23.242378+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":766,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.359981+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:27:25.359981+0000","last_peered":"2024-09-14T04:27:25.359981+0000","last_clean":"2024-09-14T04:27:25.359981+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:27:25.359981+0000","last_undegraded":"2024-09-14T04:27:25.359981+0000","last_fullsized":"2024-09-14T04:27:25.359981+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.14","version":"446'9753","reported_seq":9909,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:27:26.105600+0000","last_change":"2024-09-14T04:27:22.326058+0000","last_active":"2024-09-14T04:27:26.105600+0000","last_peered":"2024-09-14T04:27:26.105600+0000","last_clean":"2024-09-14T04:27:18.765286+0000","last_became_active":"2024-09-14T04:27:22.309237+0000","last_became_peered":"2024-09-14T04:27:22.309237+0000","last_unstale":"2024-09-14T04:27:26.105600+0000","last_undegraded":"2024-09-14T04:27:26.105600+0000","last_fullsized":"2024-09-14T04:27:26.105600+0000","mapping_epoch":443,"log_start":"412'2700","ondisk_log_start":"412'2700","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9492","last_scrub_stamp":"2024-09-14T04:27:11.031534+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:11.031534+0000","objects_scrubbed":2607,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152961024,"num_objects":2334,"num_object_clones":0,"num_object_copies":7002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4668,"num_objects_unfound":0,"num_objects_dirty":2334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":273,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,2,5],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2,5","objects":2334}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":296,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239758+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:27:23.239758+0000","last_peered":"2024-09-14T04:27:23.239758+0000","last_clean":"2024-09-14T04:27:23.239758+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:27:23.239758+0000","last_undegraded":"2024-09-14T04:27:23.239758+0000","last_fullsized":"2024-09-14T04:27:23.239758+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":34,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.231694+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:27:22.231694+0000","last_peered":"2024-09-14T04:27:22.231694+0000","last_clean":"2024-09-14T04:27:22.231694+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:27:22.231694+0000","last_undegraded":"2024-09-14T04:27:22.231694+0000","last_fullsized":"2024-09-14T04:27:22.231694+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.15","version":"445'9715","reported_seq":9845,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.157309+0000","last_change":"2024-09-14T04:27:11.636491+0000","last_active":"2024-09-14T04:27:25.157309+0000","last_peered":"2024-09-14T04:27:25.157309+0000","last_clean":"2024-09-14T04:27:25.157309+0000","last_became_active":"2024-09-14T04:27:04.679553+0000","last_became_peered":"2024-09-14T04:27:04.679553+0000","last_unstale":"2024-09-14T04:27:25.157309+0000","last_undegraded":"2024-09-14T04:27:25.157309+0000","last_fullsized":"2024-09-14T04:27:25.157309+0000","mapping_epoch":430,"log_start":"420'4100","ondisk_log_start":"420'4100","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9476","last_scrub_stamp":"2024-09-14T04:27:11.636212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:11.636212+0000","objects_scrubbed":2601,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:54:33.525805+0000","scrub_duration":0.52699493200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154402816,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":245,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":177,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.364766+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:27:25.364766+0000","last_peered":"2024-09-14T04:27:25.364766+0000","last_clean":"2024-09-14T04:27:25.364766+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:27:25.364766+0000","last_undegraded":"2024-09-14T04:27:25.364766+0000","last_fullsized":"2024-09-14T04:27:25.364766+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":520,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.242862+0000","last_change":"2024-09-14T04:27:20.298495+0000","last_active":"2024-09-14T04:27:23.242862+0000","last_peered":"2024-09-14T04:27:23.242862+0000","last_clean":"2024-09-14T04:27:23.242862+0000","last_became_active":"2024-09-14T04:27:20.297682+0000","last_became_peered":"2024-09-14T04:27:20.297682+0000","last_unstale":"2024-09-14T04:27:23.242862+0000","last_undegraded":"2024-09-14T04:27:23.242862+0000","last_fullsized":"2024-09-14T04:27:23.242862+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:02.663766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.12","version":"446'9600","reported_seq":19298,"reported_epoch":446,"state":"active+remapped+backfilling","last_fresh":"2024-09-14T04:27:26.810799+0000","last_change":"2024-09-14T04:27:20.363892+0000","last_active":"2024-09-14T04:27:26.810799+0000","last_peered":"2024-09-14T04:27:26.810799+0000","last_clean":"2024-09-14T04:27:16.175757+0000","last_became_active":"2024-09-14T04:27:18.307615+0000","last_became_peered":"2024-09-14T04:27:18.307615+0000","last_unstale":"2024-09-14T04:27:26.810799+0000","last_undegraded":"2024-09-14T04:27:26.810799+0000","last_fullsized":"2024-09-14T04:27:26.810799+0000","mapping_epoch":439,"log_start":"416'3200","ondisk_log_start":"416'3200","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"433'9342","last_scrub_stamp":"2024-09-14T04:27:12.514256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:12.514256+0000","objects_scrubbed":2588,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.39924970199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152567808,"num_objects":2328,"num_object_clones":0,"num_object_copies":6984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2506,"num_objects_unfound":0,"num_objects_dirty":2328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":260,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1133,"num_bytes_recovered":74252288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[3,4,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2328}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.234246+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:27:22.234246+0000","last_peered":"2024-09-14T04:27:22.234246+0000","last_clean":"2024-09-14T04:27:22.234246+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:27:22.234246+0000","last_undegraded":"2024-09-14T04:27:22.234246+0000","last_fullsized":"2024-09-14T04:27:22.234246+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":394,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239464+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:27:23.239464+0000","last_peered":"2024-09-14T04:27:23.239464+0000","last_clean":"2024-09-14T04:27:23.239464+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:27:23.239464+0000","last_undegraded":"2024-09-14T04:27:23.239464+0000","last_fullsized":"2024-09-14T04:27:23.239464+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.13","version":"445'9609","reported_seq":12860,"reported_epoch":445,"state":"active+remapped+backfilling","last_fresh":"2024-09-14T04:27:24.583229+0000","last_change":"2024-09-14T04:27:20.365222+0000","last_active":"2024-09-14T04:27:24.583229+0000","last_peered":"2024-09-14T04:27:24.583229+0000","last_clean":"2024-09-14T04:27:17.243789+0000","last_became_active":"2024-09-14T04:27:20.298114+0000","last_became_peered":"2024-09-14T04:27:20.298114+0000","last_unstale":"2024-09-14T04:27:24.583229+0000","last_undegraded":"2024-09-14T04:27:24.583229+0000","last_fullsized":"2024-09-14T04:27:24.583229+0000","mapping_epoch":441,"log_start":"418'3600","ondisk_log_start":"418'3600","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"431'9232","last_scrub_stamp":"2024-09-14T04:27:07.385967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.385967+0000","objects_scrubbed":2775,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":798,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156237824,"num_objects":2384,"num_object_clones":0,"num_object_copies":7152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2776,"num_objects_unfound":0,"num_objects_dirty":2384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":391,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1031,"num_bytes_recovered":67567616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2384}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.235382+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:27:22.235382+0000","last_peered":"2024-09-14T04:27:22.235382+0000","last_clean":"2024-09-14T04:27:22.235382+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:27:22.235382+0000","last_undegraded":"2024-09-14T04:27:22.235382+0000","last_fullsized":"2024-09-14T04:27:22.235382+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":403,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.239610+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:27:23.239610+0000","last_peered":"2024-09-14T04:27:23.239610+0000","last_clean":"2024-09-14T04:27:23.239610+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:27:23.239610+0000","last_undegraded":"2024-09-14T04:27:23.239610+0000","last_fullsized":"2024-09-14T04:27:23.239610+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.10","version":"446'9627","reported_seq":14905,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:26.032595+0000","last_change":"2024-09-14T04:27:20.260205+0000","last_active":"2024-09-14T04:27:26.032595+0000","last_peered":"2024-09-14T04:27:26.032595+0000","last_clean":"2024-09-14T04:27:26.032595+0000","last_became_active":"2024-09-14T04:27:20.242343+0000","last_became_peered":"2024-09-14T04:27:20.242343+0000","last_unstale":"2024-09-14T04:27:26.032595+0000","last_undegraded":"2024-09-14T04:27:26.032595+0000","last_fullsized":"2024-09-14T04:27:26.032595+0000","mapping_epoch":441,"log_start":"420'3900","ondisk_log_start":"420'3900","created":430,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"431'9216","last_scrub_stamp":"2024-09-14T04:27:07.003313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.003313+0000","objects_scrubbed":2796,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:02.114684+0000","scrub_duration":928,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155516928,"num_objects":2373,"num_object_clones":0,"num_object_copies":7119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2554,"num_bytes_recovered":167378944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":655,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.363708+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:27:25.363708+0000","last_peered":"2024-09-14T04:27:25.363708+0000","last_clean":"2024-09-14T04:27:25.363708+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:27:25.363708+0000","last_undegraded":"2024-09-14T04:27:25.363708+0000","last_fullsized":"2024-09-14T04:27:25.363708+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":326,"reported_epoch":446,"state":"active+clean","last_fresh":"2024-09-14T04:27:25.357076+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:27:25.357076+0000","last_peered":"2024-09-14T04:27:25.357076+0000","last_clean":"2024-09-14T04:27:25.357076+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:27:25.357076+0000","last_undegraded":"2024-09-14T04:27:25.357076+0000","last_fullsized":"2024-09-14T04:27:25.357076+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"446'9391","reported_seq":11725,"reported_epoch":446,"state":"active+remapped","last_fresh":"2024-09-14T04:27:27.374596+0000","last_change":"2024-09-14T04:27:20.255717+0000","last_active":"2024-09-14T04:27:27.374596+0000","last_peered":"2024-09-14T04:27:27.374596+0000","last_clean":"2024-09-14T04:27:12.693995+0000","last_became_active":"2024-09-14T04:27:15.299745+0000","last_became_peered":"2024-09-14T04:27:15.299745+0000","last_unstale":"2024-09-14T04:27:27.374596+0000","last_undegraded":"2024-09-14T04:27:27.374596+0000","last_fullsized":"2024-09-14T04:27:27.374596+0000","mapping_epoch":436,"log_start":"409'2200","ondisk_log_start":"409'2200","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'8987","last_scrub_stamp":"2024-09-14T04:27:08.864358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:08.864358+0000","objects_scrubbed":2611,"log_size":7191,"log_dups_size":0,"ondisk_log_size":7191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":787,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3066,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":424,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":2187}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":427,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-14T04:27:22.232439+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:27:22.232439+0000","last_peered":"2024-09-14T04:27:22.232439+0000","last_clean":"2024-09-14T04:27:22.232439+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:27:22.232439+0000","last_undegraded":"2024-09-14T04:27:22.232439+0000","last_fullsized":"2024-09-14T04:27:22.232439+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":548,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-14T04:27:23.246191+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:27:23.246191+0000","last_peered":"2024-09-14T04:27:23.246191+0000","last_clean":"2024-09-14T04:27:23.246191+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:27:23.246191+0000","last_undegraded":"2024-09-14T04:27:23.246191+0000","last_fullsized":"2024-09-14T04:27:23.246191+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":4905697304,"num_objects":74856,"num_object_clones":0,"num_object_copies":224568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":36477,"num_objects_unfound":0,"num_objects_dirty":74856,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":67471,"num_write_kb":2843905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5442,"num_bytes_recovered":356646912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15273570304,"data_stored":15273558088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158859,"ondisk_log_size":158859,"up":78,"acting":78,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":17064,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":22070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2426464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201842,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1340304,"kb_used_data":71048,"kb_used_omap":1,"kb_used_meta":1269246,"kb_avail":92404848,"statfs":{"total":95995035648,"available":94622564352,"internally_reserved":0,"allocated":72753152,"data_stored":69867959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1299708363},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":201863463480,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834153,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3760148,"kb_used_data":3489928,"kb_used_omap":5,"kb_used_meta":270202,"kb_avail":89985004,"statfs":{"total":95995035648,"available":92144644096,"internally_reserved":0,"allocated":3573686272,"data_stored":3570805809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":276686909},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986752,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4450060,"kb_used_data":3041604,"kb_used_omap":1,"kb_used_meta":1408382,"kb_avail":89295092,"statfs":{"total":95995035648,"available":91438174208,"internally_reserved":0,"allocated":3114602496,"data_stored":3111721080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1442183622},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748781,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2814616,"kb_used_data":1802252,"kb_used_omap":4,"kb_used_meta":1012155,"kb_avail":90930536,"statfs":{"total":95995035648,"available":93112868864,"internally_reserved":0,"allocated":1845506048,"data_stored":1842621295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1036447276},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605287002,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3290464,"kb_used_data":2725332,"kb_used_omap":10,"kb_used_meta":565045,"kb_avail":90454688,"statfs":{"total":95995035648,"available":92625600512,"internally_reserved":0,"allocated":2790739968,"data_stored":2787852211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":578606679},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468324,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2971272,"kb_used_data":1911872,"kb_used_omap":5,"kb_used_meta":1059258,"kb_avail":90773880,"statfs":{"total":95995035648,"available":92952453120,"internally_reserved":0,"allocated":1957756928,"data_stored":1954882358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1084680543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342439,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2660040,"kb_used_data":1974840,"kb_used_omap":10,"kb_used_meta":685173,"kb_avail":91085112,"statfs":{"total":95995035648,"available":93271154688,"internally_reserved":0,"allocated":2022236160,"data_stored":2019363014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":701617362},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1990459392,"data_stored":1990459392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1950158848,"data_stored":1950154776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2777874432,"data_stored":2777874432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1824788480,"data_stored":1824784408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3098284032,"data_stored":3098279960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3564765184,"data_stored":3564765184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":67239936,"data_stored":67239936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:27:33.550 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_8 pgp_num 26 2024-09-14T04:27:33.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:33.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:33.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:33.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:33.879 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:33.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:33.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:34.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:34.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:34.162 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:34.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:34.301 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:34.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:34.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:34.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:34.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:34.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:34.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:34.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:34.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:34.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:34 smithi012 ceph-mon[101180]: pgmap v1466: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 31921/207267 objects misplaced (15.401%); 16 MiB/s, 0 keys/s, 255 objects/s recovering 2024-09-14T04:27:34.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:34.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:34.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:34.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:34 smithi001 ceph-mon[112165]: pgmap v1466: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 31921/207267 objects misplaced (15.401%); 16 MiB/s, 0 keys/s, 255 objects/s recovering 2024-09-14T04:27:34.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:34 smithi001 ceph-mon[107399]: pgmap v1466: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 31921/207267 objects misplaced (15.401%); 16 MiB/s, 0 keys/s, 255 objects/s recovering 2024-09-14T04:27:35.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:35.103 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:35.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:35.242 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:35.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:35.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.427 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:35.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:35.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:35.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:35.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:35.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:35.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:35.988 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:35.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:36.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:36.129 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:36.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:36.237 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:36.238 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:36.238 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:35 smithi012 ceph-mon[101180]: pgmap v1467: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 29756/200703 objects misplaced (14.826%); 12 MiB/s, 190 objects/s recovering 2024-09-14T04:27:36.285 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:36.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:36.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:36.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:36.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[112165]: pgmap v1467: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 29756/200703 objects misplaced (14.826%); 12 MiB/s, 190 objects/s recovering 2024-09-14T04:27:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:36.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:35 smithi001 ceph-mon[107399]: pgmap v1467: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 29756/200703 objects misplaced (14.826%); 12 MiB/s, 190 objects/s recovering 2024-09-14T04:27:36.529 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:36.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:36.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:36.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:36.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:36.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:36.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:36.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:36] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:27:36.972 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:36.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:37.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:37.107 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:37.209 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:37.249 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:37.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:37.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:37.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:37.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:37.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:37.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:37.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:37.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:37.891 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:37.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:37.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:37.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:38.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:38.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:38.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:38.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:38.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:38.329 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:38.430 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:38.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:38.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:38.620 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:27:38.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:38.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:38.753 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:38.855 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:38.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:38.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:38.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[112165]: pgmap v1468: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 27647/190965 objects misplaced (14.478%); 13 MiB/s, 201 objects/s recovering 2024-09-14T04:27:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1838248857' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[107399]: pgmap v1468: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 27647/190965 objects misplaced (14.478%); 13 MiB/s, 201 objects/s recovering 2024-09-14T04:27:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1838248857' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:38.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:38 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:38.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:39.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:39.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:38 smithi012 ceph-mon[101180]: pgmap v1468: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 27647/190965 objects misplaced (14.478%); 13 MiB/s, 201 objects/s recovering 2024-09-14T04:27:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1838248857' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:38 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:27:39.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:39.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:39.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:39.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:39.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:39.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:39.423 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:39.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:39.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:39.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:39.693 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:39.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:39.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:39.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:39.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:39.950 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:39.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:39.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:39.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: osdmap e447: 8 total, 8 up, 7 in 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: osdmap e448: 8 total, 8 up, 7 in 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: osdmap e447: 8 total, 8 up, 7 in 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: osdmap e448: 8 total, 8 up, 7 in 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:40.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:40.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:40.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:27:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:27:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: osdmap e447: 8 total, 8 up, 7 in 2024-09-14T04:27:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: osdmap e448: 8 total, 8 up, 7 in 2024-09-14T04:27:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:27:40.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:40.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:40.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:40.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:40.341 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:40.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:40.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:40.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:40.603 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:40.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:40.739 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:40.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:40.871 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:40 smithi001 ceph-mon[107399]: pgmap v1471: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 27647/190965 objects misplaced (14.478%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:27:40.871 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:40 smithi001 ceph-mon[107399]: osdmap e449: 8 total, 8 up, 7 in 2024-09-14T04:27:40.871 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:40 smithi001 ceph-mon[112165]: pgmap v1471: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 27647/190965 objects misplaced (14.478%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:27:40.872 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:40 smithi001 ceph-mon[112165]: osdmap e449: 8 total, 8 up, 7 in 2024-09-14T04:27:40.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:40.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:40.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:41.023 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:40 smithi012 ceph-mon[101180]: pgmap v1471: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 108 active+clean; 3.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 27647/190965 objects misplaced (14.478%); 10 MiB/s, 166 objects/s recovering 2024-09-14T04:27:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:40 smithi012 ceph-mon[101180]: osdmap e449: 8 total, 8 up, 7 in 2024-09-14T04:27:41.124 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:41.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:41.282 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:41.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:41.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:41.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:41.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:41.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:41.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:41.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:41.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:41.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:41.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:41.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:42.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:42.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:42.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:42.247 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:42.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:42.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:42.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:42.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:42.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:42.572 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:42.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:42.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:42.824 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:42.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:42.920 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:42 smithi001 ceph-mon[112165]: pgmap v1473: 119 pgs: 1 peering, 1 active+remapped, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 24829/181257 objects misplaced (13.698%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:27:42.921 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:42 smithi001 ceph-mon[107399]: pgmap v1473: 119 pgs: 1 peering, 1 active+remapped, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 24829/181257 objects misplaced (13.698%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:27:42.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:42.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:42.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:43.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:43.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:42 smithi012 ceph-mon[101180]: pgmap v1473: 119 pgs: 1 peering, 1 active+remapped, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 24829/181257 objects misplaced (13.698%); 14 MiB/s, 227 objects/s recovering 2024-09-14T04:27:43.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:43.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:43.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:43.367 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:43.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:43.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:43.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:43.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.680 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:43.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:43.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:43.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:43.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:43.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:43.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:44.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:44.098 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:44.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:44.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:44.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:44.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:44.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:44.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:44.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:44.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.651 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:44.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:44.751 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:44.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:44.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:44.938 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:44.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:44.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:44 smithi001 ceph-mon[112165]: pgmap v1474: 119 pgs: 1 active+remapped, 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 24208/177387 objects misplaced (13.647%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:27:44.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:44 smithi001 ceph-mon[107399]: pgmap v1474: 119 pgs: 1 active+remapped, 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 24208/177387 objects misplaced (13.647%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:27:45.039 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:45.074 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:45.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:45.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:44 smithi012 ceph-mon[101180]: pgmap v1474: 119 pgs: 1 active+remapped, 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 109 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 24208/177387 objects misplaced (13.647%); 9.3 MiB/s, 148 objects/s recovering 2024-09-14T04:27:45.137 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:45.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:45.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:45.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:45.350 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:45.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:45.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:45.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:45.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:45.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:45.744 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:45.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:45.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:45.924 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:45.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:46.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:46.058 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:46.160 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:46.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:46.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:46.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:46.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:46.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:46.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:46.626 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:46 smithi001 ceph-mon[112165]: pgmap v1475: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped, 7 active+remapped+backfill_wait, 110 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 22095/170502 objects misplaced (12.959%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:27:46.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/384621438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:46.704 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:46 smithi001 ceph-mon[107399]: pgmap v1475: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped, 7 active+remapped+backfill_wait, 110 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 22095/170502 objects misplaced (12.959%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:27:46.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/384621438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:46.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:46.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:46.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:46.900 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:46.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:46.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:46] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:27:47.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:47.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:47.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:27:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:46 smithi012 ceph-mon[101180]: pgmap v1475: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped, 7 active+remapped+backfill_wait, 110 active+clean; 3.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 22095/170502 objects misplaced (12.959%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:27:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/384621438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:27:47.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.183 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:47.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:47.307 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:47.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:47.432 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:47.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:47.557 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:47.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:47.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:47.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:47.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:47.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:47.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:47 smithi001 ceph-mon[107399]: osdmap e450: 8 total, 8 up, 7 in 2024-09-14T04:27:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:47 smithi001 ceph-mon[112165]: osdmap e450: 8 total, 8 up, 7 in 2024-09-14T04:27:47.973 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:47.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:48.075 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:47 smithi012 ceph-mon[101180]: osdmap e450: 8 total, 8 up, 7 in 2024-09-14T04:27:48.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:48.195 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:27:48.195 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:27:48.195 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:27:48.196 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:27:48.197 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:27:48.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:48.267 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:48.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:48.515 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:48.551 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:48.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:48.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:48.790 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:48.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:48.919 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:48.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:48.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:48.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:48 smithi001 ceph-mon[107399]: pgmap v1477: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 20539/161418 objects misplaced (12.724%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:27:48.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:48 smithi001 ceph-mon[107399]: osdmap e451: 8 total, 8 up, 7 in 2024-09-14T04:27:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:48 smithi001 ceph-mon[112165]: pgmap v1477: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 20539/161418 objects misplaced (12.724%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:27:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:48 smithi001 ceph-mon[112165]: osdmap e451: 8 total, 8 up, 7 in 2024-09-14T04:27:49.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:48 smithi012 ceph-mon[101180]: pgmap v1477: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 20539/161418 objects misplaced (12.724%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:27:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:48 smithi012 ceph-mon[101180]: osdmap e451: 8 total, 8 up, 7 in 2024-09-14T04:27:49.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:49.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:49.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:49.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:49.337 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:49.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:49.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:49.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:49.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:49.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:49.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:49.804 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:49.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:49.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:49.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:50.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:50.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:50.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:50.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:50.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:50.365 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:50.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:50.503 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:50.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.649 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:50.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:50.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:50.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:50.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:50.927 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:50.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:50.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:50 smithi001 ceph-mon[107399]: pgmap v1479: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 20539/161418 objects misplaced (12.724%); 13 MiB/s, 200 objects/s recovering 2024-09-14T04:27:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:50 smithi001 ceph-mon[112165]: pgmap v1479: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 20539/161418 objects misplaced (12.724%); 13 MiB/s, 200 objects/s recovering 2024-09-14T04:27:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:51.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:51.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:50 smithi012 ceph-mon[101180]: pgmap v1479: 119 pgs: 1 active+remapped, 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 20539/161418 objects misplaced (12.724%); 13 MiB/s, 200 objects/s recovering 2024-09-14T04:27:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:51.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:51.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:51.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:51.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:51.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:51.482 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:51.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:51.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:51.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:51.755 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:51.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:51.857 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:51.901 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:51.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:52.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:52.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:52.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:52.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:52.287 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:52.323 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:52.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:52.425 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:52.464 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:52.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:52.600 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:52.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:52.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:52.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:52.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:52.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:52.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:52.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:52 smithi001 ceph-mon[112165]: pgmap v1480: 119 pgs: 1 active+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 18976/150411 objects misplaced (12.616%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:27:52.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:52 smithi001 ceph-mon[107399]: pgmap v1480: 119 pgs: 1 active+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 18976/150411 objects misplaced (12.616%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:27:52.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.021 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:52 smithi012 ceph-mon[101180]: pgmap v1480: 119 pgs: 1 active+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 18976/150411 objects misplaced (12.616%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:27:53.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:53.160 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:53.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:53.289 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:53.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:53.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:53.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:53.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:53.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:53.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:53.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:53.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:53.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:53.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:53.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:54.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:54.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:54.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:54.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:54.255 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:54.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:54.395 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:54.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:54.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:54.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:54.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:54.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:54.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:54.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:54.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:54.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:54.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:54.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:54 smithi001 ceph-mon[112165]: pgmap v1481: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 17146/145653 objects misplaced (11.772%); 14 MiB/s, 229 objects/s recovering 2024-09-14T04:27:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:54 smithi001 ceph-mon[112165]: osdmap e452: 8 total, 8 up, 7 in 2024-09-14T04:27:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:54 smithi001 ceph-mon[107399]: pgmap v1481: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 17146/145653 objects misplaced (11.772%); 14 MiB/s, 229 objects/s recovering 2024-09-14T04:27:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:54 smithi001 ceph-mon[107399]: osdmap e452: 8 total, 8 up, 7 in 2024-09-14T04:27:55.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:55.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:54 smithi012 ceph-mon[101180]: pgmap v1481: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 111 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 17146/145653 objects misplaced (11.772%); 14 MiB/s, 229 objects/s recovering 2024-09-14T04:27:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:54 smithi012 ceph-mon[101180]: osdmap e452: 8 total, 8 up, 7 in 2024-09-14T04:27:55.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:55.175 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:55.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:27:55.310 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:27:55.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:55.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:55.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:55.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:55.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:55.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:55.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:55.839 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:27:55.839 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:27:55.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:55.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:55.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:55.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:55.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:56.094 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:56.134 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:56.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:56.266 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 13 pgp_num to 26 2024-09-14T04:27:56.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:56.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:56.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:56.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[107399]: osdmap e453: 8 total, 8 up, 7 in 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[107399]: pgmap v1484: 119 pgs: 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 111 active+clean; 2.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 997 op/s; 15507/137733 objects misplaced (11.259%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2122924631' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[112165]: osdmap e453: 8 total, 8 up, 7 in 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[112165]: pgmap v1484: 119 pgs: 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 111 active+clean; 2.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 997 op/s; 15507/137733 objects misplaced (11.259%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2122924631' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.556 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:56.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:56 smithi012 ceph-mon[101180]: osdmap e453: 8 total, 8 up, 7 in 2024-09-14T04:27:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:56 smithi012 ceph-mon[101180]: pgmap v1484: 119 pgs: 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 111 active+clean; 2.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 997 op/s; 15507/137733 objects misplaced (11.259%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:27:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2122924631' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:56 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:27:56.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:27:56.686 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:27:56.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:27:56.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:56.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:56.917 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:56.939 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:56.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:27:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:27:56] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.51.0" 2024-09-14T04:27:57.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:57.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:57.069 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:57.115 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:27:57.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:57.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:57.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.327 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:57.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:57.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:27:57.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:57.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:57 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:27:57.462 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:57 smithi001 ceph-mon[112165]: osdmap e454: 8 total, 8 up, 7 in 2024-09-14T04:27:57.462 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:57 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:27:57.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:57 smithi001 ceph-mon[107399]: osdmap e454: 8 total, 8 up, 7 in 2024-09-14T04:27:57.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:27:57.558 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:27:57.564 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:57 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:27:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:57 smithi012 ceph-mon[101180]: osdmap e454: 8 total, 8 up, 7 in 2024-09-14T04:27:57.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:57.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:57.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:57.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:57.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:57.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:58.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.072 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:58.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:58.176 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:58.216 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:58.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:58.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:58.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:58 smithi001 ceph-mon[107399]: osdmap e455: 8 total, 8 up, 7 in 2024-09-14T04:27:58.361 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:27:58 smithi001 ceph-mon[107399]: pgmap v1487: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 15012/132225 objects misplaced (11.353%); 11 MiB/s, 169 objects/s recovering 2024-09-14T04:27:58.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:58 smithi001 ceph-mon[112165]: osdmap e455: 8 total, 8 up, 7 in 2024-09-14T04:27:58.361 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:27:58 smithi001 ceph-mon[112165]: pgmap v1487: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 15012/132225 objects misplaced (11.353%); 11 MiB/s, 169 objects/s recovering 2024-09-14T04:27:58.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:27:58.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:27:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:58 smithi012 ceph-mon[101180]: osdmap e455: 8 total, 8 up, 7 in 2024-09-14T04:27:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:27:58 smithi012 ceph-mon[101180]: pgmap v1487: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 15012/132225 objects misplaced (11.353%); 11 MiB/s, 169 objects/s recovering 2024-09-14T04:27:58.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:58.638 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:58.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:27:58.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:58.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:58.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:58.930 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:58.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:59.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:59.062 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:59.164 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:59.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:59.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:59.444 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:27:59.480 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:27:59.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:27:59.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:27:59.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:27:59.758 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:27:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:27:59.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:27:59.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:27:59.900 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:27:59.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:00.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:00.039 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:00.140 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:00.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:00.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:00.308 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:00.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:00.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:00.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:00.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:00.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:00 smithi012 ceph-mon[101180]: pgmap v1488: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 13198/130095 objects misplaced (10.145%); 17 MiB/s, 265 objects/s recovering 2024-09-14T04:28:00.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:00 smithi001 ceph-mon[107399]: pgmap v1488: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 13198/130095 objects misplaced (10.145%); 17 MiB/s, 265 objects/s recovering 2024-09-14T04:28:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:00 smithi001 ceph-mon[112165]: pgmap v1488: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 13198/130095 objects misplaced (10.145%); 17 MiB/s, 265 objects/s recovering 2024-09-14T04:28:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:00.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:00.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:00.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:00.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:00.985 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:01.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:01.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:01.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:01.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:01.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:01.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:01.436 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:01.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:01.578 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:01.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:01.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:01.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:01.856 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:01.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:01.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:01.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:02.095 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:02.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:02.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:02.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:02.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:02.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:02.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:02.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:02.499 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 7] out_osds: [6] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:28:02.500 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:28:02.500 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-14T04:28:02.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd in 6 2024-09-14T04:28:02.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:02.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:02.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:02 smithi001 ceph-mon[107399]: pgmap v1489: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 11158/121461 objects misplaced (9.186%); 23 MiB/s, 364 objects/s recovering 2024-09-14T04:28:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:02 smithi001 ceph-mon[112165]: pgmap v1489: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 11158/121461 objects misplaced (9.186%); 23 MiB/s, 364 objects/s recovering 2024-09-14T04:28:02.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:02.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:02.833 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:02.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:02.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:02.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:02.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:03.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:02 smithi012 ceph-mon[101180]: pgmap v1489: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 11158/121461 objects misplaced (9.186%); 23 MiB/s, 364 objects/s recovering 2024-09-14T04:28:03.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:03.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:03.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:03.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:03.285 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:28:03.285 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:28:03.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:03.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:03.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:03.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:03.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:03.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:03.692 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:03.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:03.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:03.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:03.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:03.945 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:03.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:04.002 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:04.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:04.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:04.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:04.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:04.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:04.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:04.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:04.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:04.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:04.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:28:04.637 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:04.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:04.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:04.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[107399]: pgmap v1490: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 10749/116781 objects misplaced (9.204%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[107399]: osdmap e456: 8 total, 8 up, 7 in 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[112165]: pgmap v1490: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 10749/116781 objects misplaced (9.204%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[112165]: osdmap e456: 8 total, 8 up, 7 in 2024-09-14T04:28:04.864 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:04 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:04.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:04.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:04.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:05.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:05.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:04 smithi012 ceph-mon[101180]: pgmap v1490: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 10749/116781 objects misplaced (9.204%); 18 MiB/s, 287 objects/s recovering 2024-09-14T04:28:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:04 smithi012 ceph-mon[101180]: osdmap e456: 8 total, 8 up, 7 in 2024-09-14T04:28:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:04 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:05.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:05.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:05.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:05.243 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:28:05.274 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:05.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:05.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:05.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:05.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:05.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:05.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[112165]: osdmap e457: 8 total, 8 up, 7 in 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:28:05.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:05 smithi001 ceph-mon[107399]: osdmap e457: 8 total, 8 up, 7 in 2024-09-14T04:28:05.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:05.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:05.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:05.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:05.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:05.962 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:06.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:06.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:05 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1137363633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:28:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:05 smithi012 ceph-mon[101180]: osdmap e457: 8 total, 8 up, 7 in 2024-09-14T04:28:06.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:06.145 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:06.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:06.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:06.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:06.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:06.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:06.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:06.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:06.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:06.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:06 smithi001 ceph-mon[112165]: pgmap v1493: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 8668/107175 objects misplaced (8.088%); 24 MiB/s, 379 objects/s recovering 2024-09-14T04:28:06.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:06 smithi001 ceph-mon[107399]: pgmap v1493: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 8668/107175 objects misplaced (8.088%); 24 MiB/s, 379 objects/s recovering 2024-09-14T04:28:06.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:06.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:06.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:06.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:06] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-09-14T04:28:06.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:06.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:06 smithi012 ceph-mon[101180]: pgmap v1493: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 8668/107175 objects misplaced (8.088%); 24 MiB/s, 379 objects/s recovering 2024-09-14T04:28:07.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:07.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:07.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:07.251 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:07.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:07.375 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:07.477 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:07.500 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:07.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:07.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:07.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:07.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:07.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:07.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:07.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:08.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:08.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:08.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:08.195 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:08.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:08.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:08.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:08.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:08.483 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:28:08.520 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:08.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:08.578 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:28:08.578 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:28:08.602 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:08.618 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:28:08.618 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:28:08.655 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:28:08.656 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:28:08.657 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:08 smithi001 ceph-mon[112165]: pgmap v1494: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 8394/101304 objects misplaced (8.286%); 18 MiB/s, 288 objects/s recovering 2024-09-14T04:28:08.657 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1630640797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:08.657 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:08 smithi001 ceph-mon[107399]: pgmap v1494: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 8394/101304 objects misplaced (8.286%); 18 MiB/s, 288 objects/s recovering 2024-09-14T04:28:08.657 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1630640797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:08.694 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:28:08.695 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:28:08.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:08.731 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:28:08.731 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:28:08.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:08.763 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:28:08.764 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:28:08.792 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:28:08.793 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:28:08.827 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:28:08.827 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:28:08.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:08.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:08.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:08.897 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:08 smithi012 ceph-mon[101180]: pgmap v1494: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 8394/101304 objects misplaced (8.286%); 18 MiB/s, 288 objects/s recovering 2024-09-14T04:28:08.897 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1630640797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:08.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:09.014 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:09.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:09.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:09.154 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:09.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:09.291 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:09.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:09.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:09.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:09.587 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:09.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:09.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:09.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:09.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:09.869 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:09.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:09.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:09.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:09.994 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:10.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:10.140 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:10.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:10.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:10.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:10.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:10.460 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:10.562 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:10.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:10.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:10.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:10.748 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:10.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:10.889 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:10.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:10.892 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:10 smithi001 ceph-mon[112165]: pgmap v1495: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 8132/99339 objects misplaced (8.186%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:10.892 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:10 smithi001 ceph-mon[107399]: pgmap v1495: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 8132/99339 objects misplaced (8.186%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:10.991 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:10.992 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:10 smithi012 ceph-mon[101180]: pgmap v1495: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 8132/99339 objects misplaced (8.186%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:11.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:11.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:11.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:11.277 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.350 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:11.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:11.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:11.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:11.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:11.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:11.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:11.767 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:11.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:11.895 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:11.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:11.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:11.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:12.046 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:12.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:12.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:12.178 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:12.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:12.310 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:12.412 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:12.448 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:12.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:12.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.573 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:12.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:12.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:12.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:28:12.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:12.832 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:12.909 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:12 smithi001 ceph-mon[112165]: pgmap v1496: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 6647/90555 objects misplaced (7.340%); 16 MiB/s, 252 objects/s recovering 2024-09-14T04:28:12.910 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:12 smithi001 ceph-mon[107399]: pgmap v1496: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 6647/90555 objects misplaced (7.340%); 16 MiB/s, 252 objects/s recovering 2024-09-14T04:28:12.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:12.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:12.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:13.062 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.093 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:13.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:12 smithi012 ceph-mon[101180]: pgmap v1496: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 6647/90555 objects misplaced (7.340%); 16 MiB/s, 252 objects/s recovering 2024-09-14T04:28:13.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:13.219 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:13.321 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:13.344 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:13.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:13.470 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:13.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:13.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:13.596 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:13.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:13.733 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:13.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:13.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:13.870 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:13.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:13.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:13 smithi001 ceph-mon[112165]: osdmap e458: 8 total, 8 up, 7 in 2024-09-14T04:28:13.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:13 smithi001 ceph-mon[112165]: osdmap e459: 8 total, 8 up, 7 in 2024-09-14T04:28:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:13 smithi001 ceph-mon[107399]: osdmap e458: 8 total, 8 up, 7 in 2024-09-14T04:28:13.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:13 smithi001 ceph-mon[107399]: osdmap e459: 8 total, 8 up, 7 in 2024-09-14T04:28:13.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:14.018 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:13 smithi012 ceph-mon[101180]: osdmap e458: 8 total, 8 up, 7 in 2024-09-14T04:28:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:13 smithi012 ceph-mon[101180]: osdmap e459: 8 total, 8 up, 7 in 2024-09-14T04:28:14.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:14.165 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:14.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:14.267 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:14.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:14.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:14.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:14.442 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:14.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:14.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:14.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.594 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:14.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:14.695 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:14.716 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:14 smithi001 ceph-mon[107399]: pgmap v1498: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 6339/86796 objects misplaced (7.303%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:28:14.716 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:14 smithi001 ceph-mon[112165]: pgmap v1498: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 6339/86796 objects misplaced (7.303%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:28:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.730 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:14.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:14.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:14.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:14.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:14.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:14.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:15.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:14 smithi012 ceph-mon[101180]: pgmap v1498: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 6339/86796 objects misplaced (7.303%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:28:15.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:15.166 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:15.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:15.298 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:15.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:15.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:15.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.581 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:15.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:15.718 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:15.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:15.851 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:15.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:15.953 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:15.999 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:16.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:16.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:16.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:16.301 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:16.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:16.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:16.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:16.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:16.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:16.728 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:16.830 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:16.872 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:16.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: pgmap v1500: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 5231/78168 objects misplaced (6.692%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: pgmap v1500: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 5231/78168 objects misplaced (6.692%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:16.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:16] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-09-14T04:28:16.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:17.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: pgmap v1500: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 5231/78168 objects misplaced (6.692%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:16 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:17.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:17.171 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:17.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:17.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:17.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:17.433 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145645 12.c 12:3996658d:::benchmark_data_smithi001_255951_object145644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 510.95199466000003, 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5001031949999999, 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.434 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145645 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447089+0000", 2024-09-14T04:28:17.435 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447090+0000", 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5370000000000001e-06 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.436 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447116+0000", 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5673999999999998e-05 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447128+0000", 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1878e-05 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.437 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.447136+0000", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0989999999999999e-06 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.451501+0000", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004364723 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:46.451511+0000", 2024-09-14T04:28:17.438 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0339999999999999e-05 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.436184+0000", 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98467291999999995 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.436197+0000", 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2951e-05 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.439 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940260+0000", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.504063127 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940396+0000", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135499 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940581+0000", 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018567799999999999 2024-09-14T04:28:17.440 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944744+0000", 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041623989999999998 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.946037+0000", 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012937890000000001 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.441 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947043+0000", 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001005508 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947101+0000", 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8458000000000002e-05 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.442 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.947192+0000", 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0614999999999999e-05 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145621 12.a 12:5542fd65:::benchmark_data_smithi001_255951_object145620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 509.95806343800001, 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4993388519999999, 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.443 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145621 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.444 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441020+0000", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441022+0000", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6819999999999999e-06 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441434+0000", 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00041247899999999999 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.445 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441451+0000", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6762e-05 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441460+0000", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8799999999999997e-06 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441708+0000", 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024832400000000002 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.446 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441719+0000", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0411000000000001e-05 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940283+0000", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4985643479999999 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940359+0000", 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5965999999999997e-05 2024-09-14T04:28:17.447 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145627 12.a 12:5df13aaa:::benchmark_data_smithi001_255951_object145626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 509.95760479299997, 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.499140605, 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.448 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145627 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:28:17.449 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441479+0000", 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3799999999999996e-07 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441541+0000", 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1072000000000001e-05 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.450 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441549+0000", 2024-09-14T04:28:17.451 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4489999999999999e-06 2024-09-14T04:28:17.451 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.451 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.451 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441555+0000", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1689999999999997e-06 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441864+0000", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030918299999999998 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441872+0000", 2024-09-14T04:28:17.452 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1110000000000008e-06 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940553+0000", 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.498680727 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940619+0000", 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6155999999999995e-05 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.453 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:145651 12.a 12:5b322833:::benchmark_data_smithi001_255951_object145650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 509.957519784, 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5069970850000001, 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 145651 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.454 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.455 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441564+0000", 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1699999999999998e-07 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441611+0000", 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6927e-05 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441620+0000", 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2110000000000004e-06 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.456 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441627+0000", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2230000000000004e-06 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441897+0000", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00027031800000000002 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:47.441904+0000", 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7399999999999998e-06 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.457 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940646+0000", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.498742332 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940762+0000", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115798 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.940921+0000", 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015967999999999999 2024-09-14T04:28:17.458 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.944358+0000", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034363340000000001 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.946696+0000", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002338082 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948409+0000", 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0017135200000000001 2024-09-14T04:28:17.459 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948463+0000", 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4008999999999997e-05 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:19:48.948561+0000", 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7393999999999996e-05 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.460 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243554 12.3 12:c3bf76db:::benchmark_data_smithi001_255951_object97894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 366.615469131, 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7736291980000001, 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243554 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.461 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783614+0000", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783616+0000", 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0080000000000001e-06 2024-09-14T04:28:17.462 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783624+0000", 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1389999999999998e-06 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783641+0000", 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7342000000000001e-05 2024-09-14T04:28:17.463 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783648+0000", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0720000000000002e-06 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783722+0000", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4325000000000004e-05 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:10.783737+0000", 2024-09-14T04:28:17.464 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4562000000000001e-05 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313440+0000", 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.52970318299999997 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313452+0000", 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2206e-05 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.465 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.537509+0000", 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2240570070000001 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.537645+0000", 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000136141 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.538767+0000", 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001121976 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.466 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.547961+0000", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0091935539999999996 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.550934+0000", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029733049999999999 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554104+0000", 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00316972 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.467 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557038+0000", 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029345489999999998 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.468 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557110+0000", 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1348999999999997e-05 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557244+0000", 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013375999999999999 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.469 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243561 12.3 12:c5ad174e:::benchmark_data_smithi001_255951_object97901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 366.21984503300001, 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3783672410000001, 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243561 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.470 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179238+0000", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179241+0000", 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2689999999999998e-06 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.471 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179256+0000", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5534000000000001e-05 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179279+0000", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2751e-05 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179290+0000", 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0593e-05 2024-09-14T04:28:17.472 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179365+0000", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5424000000000002e-05 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.179374+0000", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3139999999999993e-06 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313477+0000", 2024-09-14T04:28:17.473 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13410250800000001 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313485+0000", 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8169999999999999e-06 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.547996+0000", 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2345118159999999 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.474 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.548122+0000", 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000125764 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.548263+0000", 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014107900000000001 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554033+0000", 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057697720000000003 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.475 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554076+0000", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2695999999999999e-05 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554110+0000", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4564999999999999e-05 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557438+0000", 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00332775 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.476 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557494+0000", 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6064999999999999e-05 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557606+0000", 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000111524 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.477 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243562 12.3 12:c85133a2:::benchmark_data_smithi001_255951_object97902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 366.12466812399998, 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.283526199, 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243562 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.478 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274415+0000", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274418+0000", 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2330000000000001e-06 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.479 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274426+0000", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6449999999999994e-06 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274448+0000", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1542000000000001e-05 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274458+0000", 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:28:17.480 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274542+0000", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3781000000000003e-05 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.274555+0000", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2846e-05 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313507+0000", 2024-09-14T04:28:17.481 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.038952458000000002 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.313515+0000", 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.550967+0000", 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2374517810000001 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.482 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.551087+0000", 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011969 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.551225+0000", 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013870899999999999 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.554116+0000", 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028902979999999999 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.483 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557298+0000", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031826139999999998 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557347+0000", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9183e-05 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557664+0000", 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000316466 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.484 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557722+0000", 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8576999999999998e-05 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.557942+0000", 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021917900000000001 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.485 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243568 12.e 12:7f8cd4cb:::benchmark_data_smithi001_255951_object97908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 366.01440440200003, 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.245202098, 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243568 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.486 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384679+0000", 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.487 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384680+0000", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4100000000000001e-06 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384684+0000", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9720000000000003e-06 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384696+0000", 2024-09-14T04:28:17.488 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1674999999999999e-05 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384702+0000", 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9660000000000001e-06 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384764+0000", 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2053999999999999e-05 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.489 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.384778+0000", 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3574000000000001e-05 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560158+0000", 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.175380281 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560276+0000", 2024-09-14T04:28:17.490 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000117953 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.560747+0000", 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00047152199999999999 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626865+0000", 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.066117457000000004 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.491 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626930+0000", 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5208000000000003e-05 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.626980+0000", 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9792e-05 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629676+0000", 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002696155 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.492 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629740+0000", 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4349000000000003e-05 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629881+0000", 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014072999999999999 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.493 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243569 12.e 12:72fca126:::benchmark_data_smithi001_255951_object97909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 366.01213215899998, 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2435670029999999, 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243569 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.494 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386951+0000", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386952+0000", 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1349999999999999e-06 2024-09-14T04:28:17.495 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386957+0000", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3080000000000001e-06 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386967+0000", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0412e-05 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.386974+0000", 2024-09-14T04:28:17.496 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3509999999999998e-06 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.387029+0000", 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5467999999999998e-05 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:11.387036+0000", 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7089999999999996e-06 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.497 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627054+0000", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.240018735 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627181+0000", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012620700000000001 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.627317+0000", 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013682200000000001 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.498 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629892+0000", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025746620000000001 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.629931+0000", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8968999999999999e-05 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630135+0000", 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000203799 2024-09-14T04:28:17.499 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630243+0000", 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108099 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630305+0000", 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1724999999999996e-05 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.500 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:12.630518+0000", 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021360199999999999 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254264 12.16 12:6e91794d:::benchmark_data_smithi001_255951_object108604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.80737660400001, 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.075071672, 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.501 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254264 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591707+0000", 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.502 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591709+0000", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8759999999999999e-06 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591717+0000", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9489999999999996e-06 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591736+0000", 2024-09-14T04:28:17.503 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9528999999999999e-05 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591745+0000", 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6300000000000004e-06 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591841+0000", 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.6379000000000007e-05 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.504 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.591857+0000", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6163000000000001e-05 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.663671+0000", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0718133569999999 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.663838+0000", 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000167651 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.505 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.507 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.664192+0000", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035332499999999998 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666120+0000", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001928443 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666342+0000", 2024-09-14T04:28:17.508 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022153200000000001 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666418+0000", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5950000000000003e-05 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666483+0000", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5798999999999998e-05 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666553+0000", 2024-09-14T04:28:17.509 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9492000000000005e-05 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666629+0000", 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6198000000000005e-05 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666779+0000", 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014939900000000001 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.510 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254293 12.6 12:60fdceba:::benchmark_data_smithi001_255951_object108633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.713253146, 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0043203650000001, 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254293 2024-09-14T04:28:17.511 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685830+0000", 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.512 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685832+0000", 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.296e-06 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685836+0000", 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4320000000000001e-06 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685849+0000", 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2768e-05 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.513 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685854+0000", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0640000000000001e-06 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.685999+0000", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014539800000000001 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.686009+0000", 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0192e-05 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.514 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676321+0000", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99031173400000005 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676450+0000", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012866599999999999 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676770+0000", 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000320501 2024-09-14T04:28:17.515 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681206+0000", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044353329999999996 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681282+0000", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6676999999999998e-05 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687652+0000", 2024-09-14T04:28:17.516 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063695410000000003 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687722+0000", 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0320000000000004e-05 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689967+0000", 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022448670000000002 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.517 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.690027+0000", 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.029e-05 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.690151+0000", 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012328600000000001 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254316 12.16 12:6a29b3f6:::benchmark_data_smithi001_255951_object108656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:28:17.518 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.67399903299997, 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94999470500000005, 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254316 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:28:17.519 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725084+0000", 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725086+0000", 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2780000000000001e-06 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.520 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725090+0000", 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3069999999999996e-06 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725102+0000", 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1569999999999999e-05 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725108+0000", 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.083e-06 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.521 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725171+0000", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3691999999999999e-05 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.725185+0000", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3451e-05 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666839+0000", 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94165448399999996 2024-09-14T04:28:17.522 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.666979+0000", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013989499999999999 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.667286+0000", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000306674 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670323+0000", 2024-09-14T04:28:17.523 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003037414 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670371+0000", 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7908000000000003e-05 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674661+0000", 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042897500000000002 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.524 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674841+0000", 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000179617 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674883+0000", 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2491999999999999e-05 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674944+0000", 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1408000000000006e-05 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.525 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675079+0000", 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013468200000000001 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254322 12.6 12:672ca547:::benchmark_data_smithi001_255951_object108662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.65938467000001, 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95668792300000005, 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.526 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254322 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739699+0000", 2024-09-14T04:28:17.527 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739700+0000", 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5099999999999999e-07 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739703+0000", 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.4680000000000001e-06 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.528 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739712+0000", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4989999999999994e-06 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739716+0000", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6800000000000001e-06 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739768+0000", 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2181000000000001e-05 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.529 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.739779+0000", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0188e-05 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681316+0000", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94153731699999998 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681452+0000", 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001359 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.530 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.681774+0000", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00032192599999999999 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687571+0000", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057971990000000003 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689807+0000", 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022359540000000001 2024-09-14T04:28:17.531 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.692137+0000", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023296929999999999 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.694001+0000", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001864276 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696206+0000", 2024-09-14T04:28:17.532 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022052460000000001 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696262+0000", 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5491000000000001e-05 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696387+0000", 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012515399999999999 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.533 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254327 12.6 12:63c13416:::benchmark_data_smithi001_255951_object108667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.64835173300003, 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94596290800000005, 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254327 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.534 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750732+0000", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750734+0000", 2024-09-14T04:28:17.535 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7630000000000001e-06 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750740+0000", 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7669999999999998e-06 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750757+0000", 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6484999999999999e-05 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.536 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750765+0000", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0520000000000002e-06 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750830+0000", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5215000000000005e-05 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.750844+0000", 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3781000000000001e-05 2024-09-14T04:28:17.537 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.683972+0000", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93312859599999998 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.684092+0000", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0001199 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.684372+0000", 2024-09-14T04:28:17.538 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000279354 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.687687+0000", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033157579999999998 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.689860+0000", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021729589999999999 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.692143+0000", 2024-09-14T04:28:17.539 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022829959999999998 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.694045+0000", 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001901944 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696493+0000", 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024481490000000002 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.540 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696556+0000", 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2799999999999995e-05 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.696695+0000", 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013838899999999999 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254363 12.16 12:6b264e56:::benchmark_data_smithi001_255951_object108703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.541 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.55938112199999, 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.83572432600000002, 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254363 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.542 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839702+0000", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839704+0000", 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.578e-06 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.543 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839709+0000", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6129999999999997e-06 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839722+0000", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.348e-05 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839730+0000", 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.405e-06 2024-09-14T04:28:17.544 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839802+0000", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1544999999999993e-05 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.839810+0000", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8480000000000001e-06 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.668751+0000", 2024-09-14T04:28:17.545 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.82894146000000002 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.668941+0000", 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018943000000000001 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.669225+0000", 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00028416599999999998 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.546 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674646+0000", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005421131 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674668+0000", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1801999999999999e-05 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674786+0000", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011822799999999999 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.547 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675213+0000", 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00042697699999999999 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675255+0000", 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1730999999999997e-05 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675308+0000", 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3114e-05 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.548 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675427+0000", 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118818 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:254376 12.16 12:684dc283:::benchmark_data_smithi001_255951_object108716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.54205377900001, 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.81920720899999999, 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.549 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 254376 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.550 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857030+0000", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857031+0000", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1599999999999999e-06 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857034+0000", 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5889999999999999e-06 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.551 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857047+0000", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2252000000000001e-05 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857053+0000", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6019999999999996e-06 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857122+0000", 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8534999999999993e-05 2024-09-14T04:28:17.552 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:04.857135+0000", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3030000000000001e-05 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670546+0000", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.81341160700000004 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.670687+0000", 2024-09-14T04:28:17.553 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014010800000000001 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6,7", 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.671031+0000", 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034398700000000003 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.674673+0000", 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036429240000000001 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.554 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675122+0000", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00044859499999999998 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675165+0000", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3022999999999998e-05 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675510+0000", 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00034440300000000002 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.555 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675550+0000", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0927999999999998e-05 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.675602+0000", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1935000000000003e-05 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:05.676237+0000", 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00063453100000000003 2024-09-14T04:28:17.556 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:176033 13.12 13:490646d2:::benchmark_data_smithi001_306460_object57789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:27:39.345054+0000", 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 38.054029364000002, 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.904171379, 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:17.557 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 176033 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345054+0000", 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345054+0000", 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.558 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345055+0000", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.2800000000000005e-07 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345060+0000", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.532e-06 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345073+0000", 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2968e-05 2024-09-14T04:28:17.559 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345078+0000", 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0599999999999998e-06 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345205+0000", 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000127069 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.560 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.345215+0000", 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0392e-05 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.244354+0000", 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.89913903900000003 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.244469+0000", 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000114584 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.561 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.244753+0000", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00028427699999999999 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.248992+0000", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042392599999999999 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249001+0000", 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4880000000000007e-06 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.562 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249119+0000", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011843199999999999 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249181+0000", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1465999999999997e-05 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249225+0000", 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4883999999999999e-05 2024-09-14T04:28:17.563 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:176090 13.12 13:4ab8ad3e:::benchmark_data_smithi001_306460_object57846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:27:39.396556+0000", 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 38.002527614999998, 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.85595614200000003, 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:17.564 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 176090 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396556+0000", 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396556+0000", 2024-09-14T04:28:17.565 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396557+0000", 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0300000000000001e-06 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396561+0000", 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6210000000000001e-06 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.566 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396570+0000", 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9599999999999995e-06 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396578+0000", 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5830000000000001e-06 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396677+0000", 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8697999999999996e-05 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.567 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.396756+0000", 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9488999999999997e-05 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.246461+0000", 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.84970464999999995 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.568 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.246528+0000", 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7249000000000006e-05 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.246720+0000", 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000192306 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249280+0000", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0025590539999999998 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249288+0000", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.789e-06 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252439+0000", 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031510240000000001 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252470+0000", 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0468e-05 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252512+0000", 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2221000000000001e-05 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.571 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:176108 13.12 13:4f09b2de:::benchmark_data_smithi001_306460_object57864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:27:39.470122+0000", 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 37.928961856999997, 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78258515200000001, 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 176108 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470122+0000", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470122+0000", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470123+0000", 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2160000000000001e-06 2024-09-14T04:28:17.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470127+0000", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0770000000000001e-06 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470138+0000", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1049999999999999e-05 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470145+0000", 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6579999999999996e-06 2024-09-14T04:28:17.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470196+0000", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1116000000000003e-05 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470205+0000", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3130000000000004e-06 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.247776+0000", 2024-09-14T04:28:17.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.77757056300000005 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.247857+0000", 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1799999999999996e-05 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.248014+0000", 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015659300000000001 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249291+0000", 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012769050000000001 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249340+0000", 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9177e-05 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252622+0000", 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032818439999999999 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252670+0000", 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8353999999999998e-05 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252707+0000", 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6486000000000003e-05 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:176110 13.12 13:4b856c46:::benchmark_data_smithi001_306460_object57866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-09-14T04:28:17.578 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:27:39.470164+0000", 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 37.928919727999997, 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78277001899999998, 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 176110 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:17.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470164+0000", 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470164+0000", 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470164+0000", 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8699999999999995e-07 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.580 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470167+0000", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.7930000000000002e-06 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470175+0000", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0120000000000005e-06 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470183+0000", 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7349999999999996e-06 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.581 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470226+0000", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3409999999999999e-05 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:39.470232+0000", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6450000000000002e-06 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249389+0000", 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.77915665499999998 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249497+0000", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108361 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.249636+0000", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000139419 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252568+0000", 2024-09-14T04:28:17.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0029316070000000001 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252580+0000", 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2103000000000001e-05 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252823+0000", 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000243339 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:17.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252892+0000", 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8567999999999998e-05 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:27:40.252934+0000", 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1785000000000001e-05 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:17.585 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:28:17.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:17.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:17.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:17.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:17.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:17.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:17.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:17.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:18.000 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:18.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:18.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:18.240 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:18.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:18.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:18.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:18.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:28:18.104+0000 7fbfffcba640 -1 mon.a@0(leader).osd e459 definitely_dead 0 2024-09-14T04:28:18.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:18.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:18.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:18.705 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:18.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:18.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:18.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:18.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:18.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:18.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[107399]: pgmap v1501: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 977 op/s; 5231/75864 objects misplaced (6.895%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:18.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1149820109' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:18.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[112165]: pgmap v1501: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 977 op/s; 5231/75864 objects misplaced (6.895%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1149820109' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:19.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:19.082 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:18 smithi012 ceph-mon[101180]: pgmap v1501: 119 pgs: 4 active+remapped+backfill_wait, 115 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 977 op/s; 5231/75864 objects misplaced (6.895%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:28:19.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1149820109' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:19.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-14T04:28:19.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:19.207 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:19.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:19.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:19.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:19.470 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:19.572 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:19.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:19.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:19.719 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:19.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:19.844 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:19.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:19.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:19.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-14T04:28:19.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[107399]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[107399]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[107399]: pgmap v1504: 119 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 3761/69435 objects misplaced (5.417%); 14 MiB/s, 221 objects/s recovering 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[112165]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[112165]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:28:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:19 smithi001 ceph-mon[112165]: pgmap v1504: 119 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 3761/69435 objects misplaced (5.417%); 14 MiB/s, 221 objects/s recovering 2024-09-14T04:28:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:19.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:20.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:20.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:19 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-14T04:28:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:19 smithi012 ceph-mon[101180]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:28:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:19 smithi012 ceph-mon[101180]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:28:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:19 smithi012 ceph-mon[101180]: pgmap v1504: 119 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 3761/69435 objects misplaced (5.417%); 14 MiB/s, 221 objects/s recovering 2024-09-14T04:28:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:20.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:20.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:20.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:20.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:20.347 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:20.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:20.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:20.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:20.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:20.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:20.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:20.698 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:20.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:20.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:20.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:20.853 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:20.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:20.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.14 continuing backfill to osd.0 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:20.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.14 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11435 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'13380 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.14 continuing backfill to osd.7 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.17 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11374 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: 13.f continuing backfill to osd.7 from (424'6150,459'13380] 13:f9ec98ce:::benchmark_data_smithi001_306460_object116904:head to 459'13380 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[107399]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.14 continuing backfill to osd.0 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.14 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11435 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'13380 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.14 continuing backfill to osd.7 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.17 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11374 2024-09-14T04:28:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: 13.f continuing backfill to osd.7 from (424'6150,459'13380] 13:f9ec98ce:::benchmark_data_smithi001_306460_object116904:head to 459'13380 2024-09-14T04:28:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:20 smithi001 ceph-mon[112165]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:28:20.991 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:20.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:21.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.14 continuing backfill to osd.0 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.14 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11435 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'13380 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.14 continuing backfill to osd.7 from (412'2700,459'11435] MIN to 459'11435 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.17 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11374 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: 13.f continuing backfill to osd.7 from (424'6150,459'13380] 13:f9ec98ce:::benchmark_data_smithi001_306460_object116904:head to 459'13380 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:20 smithi012 ceph-mon[101180]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:28:21.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:21.226 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:21.259 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:21.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:21.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:21.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:21.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:21.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:21.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:21.771 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:21.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:21.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:21.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:21.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:21.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[112165]: 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11405 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[112165]: 13.18 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11112 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[112165]: 13.11 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10942 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[112165]: 13.8 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11123 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[112165]: pgmap v1506: 119 pgs: 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 110 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 3488/62067 objects misplaced (5.620%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[107399]: 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11405 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[107399]: 13.18 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11112 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[107399]: 13.11 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10942 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[107399]: 13.8 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11123 2024-09-14T04:28:21.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:21 smithi001 ceph-mon[107399]: pgmap v1506: 119 pgs: 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 110 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 3488/62067 objects misplaced (5.620%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:28:22.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:22.077 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:22.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:21 smithi012 ceph-mon[101180]: 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11405 2024-09-14T04:28:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:21 smithi012 ceph-mon[101180]: 13.18 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11112 2024-09-14T04:28:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:21 smithi012 ceph-mon[101180]: 13.11 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10942 2024-09-14T04:28:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:21 smithi012 ceph-mon[101180]: 13.8 starting backfill to osd.6 from (0'0,0'0] MAX to 459'11123 2024-09-14T04:28:22.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:21 smithi012 ceph-mon[101180]: pgmap v1506: 119 pgs: 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 110 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 3488/62067 objects misplaced (5.620%); 15 MiB/s, 240 objects/s recovering 2024-09-14T04:28:22.178 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:22.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:22.301 INFO:teuthology.orchestra.run.smithi001.stderr:marked in osd.6. 2024-09-14T04:28:22.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:22.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:22.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:22.458 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:22.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:22.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:22.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:22.712 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:22.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:22.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:22.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:22.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:22.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:22 smithi001 ceph-mon[107399]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:28:22.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:22 smithi001 ceph-mon[112165]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:28:22.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:22.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:22 smithi012 ceph-mon[101180]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:28:23.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:23.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:23.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:23.267 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:23.369 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:23.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:23.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:23.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:23.540 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:23.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:23.682 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:23.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:23.823 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:23.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:23.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:23.948 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:23.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:23.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:23.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:23 smithi001 ceph-mon[112165]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:28:23.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:23 smithi001 ceph-mon[112165]: pgmap v1509: 119 pgs: 1 peering, 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 107 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5187/60003 objects misplaced (8.645%); 17 MiB/s, 266 objects/s recovering 2024-09-14T04:28:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:23 smithi001 ceph-mon[107399]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:28:23.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:23 smithi001 ceph-mon[107399]: pgmap v1509: 119 pgs: 1 peering, 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 107 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5187/60003 objects misplaced (8.645%); 17 MiB/s, 266 objects/s recovering 2024-09-14T04:28:24.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:24.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:23 smithi012 ceph-mon[101180]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:28:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:23 smithi012 ceph-mon[101180]: pgmap v1509: 119 pgs: 1 peering, 2 active+undersized+remapped, 3 remapped+peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 107 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5187/60003 objects misplaced (8.645%); 17 MiB/s, 266 objects/s recovering 2024-09-14T04:28:24.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:24.236 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:24.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:24.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:24.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:24.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:24.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:24.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:24.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:24.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:24.922 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:24.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:25.024 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:25.058 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:25.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:25.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:25.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:25.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:25.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:25.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:25.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:25.593 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:25.633 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:25.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:25.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:25.772 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:25.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:25.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:25.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:25.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:25.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:26.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:26.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:26.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:26.161 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:26.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:26.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:26.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:26.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:26.440 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:28:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:26.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:26.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:26.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:26.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:26.632 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:28:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[107399]: pgmap v1510: 119 pgs: 1 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 109 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 5947/54789 objects misplaced (10.854%); 20 MiB/s, 314 objects/s recovering 2024-09-14T04:28:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3574973634' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:26.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[112165]: pgmap v1510: 119 pgs: 1 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 109 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 5947/54789 objects misplaced (10.854%); 20 MiB/s, 314 objects/s recovering 2024-09-14T04:28:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3574973634' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:26.704 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:26.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:26.778 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:26.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:26.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:26.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:26.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:26.965 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:26] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.51.0" 2024-09-14T04:28:27.003 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-14T04:28:27.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.059 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:27.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:26 smithi012 ceph-mon[101180]: pgmap v1510: 119 pgs: 1 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 109 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 5947/54789 objects misplaced (10.854%); 20 MiB/s, 314 objects/s recovering 2024-09-14T04:28:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:26 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3574973634' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:26 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:27.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:27.175 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:27.205 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:27.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:27.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.350 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:27.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:27.451 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:27.492 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:27.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:27.631 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:27.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:27.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:27.874 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[107399]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:27.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:27 smithi001 ceph-mon[112165]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:28:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:27.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:27.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:28.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:28.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:28.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:27 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:28:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:27 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:28:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:27 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:27 smithi012 ceph-mon[101180]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:28:28.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:28.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:28.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:28.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:28.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:28.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:28.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:28.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:28.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:28.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:28.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:28.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:28.893 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:28.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:28 smithi001 ceph-mon[107399]: pgmap v1512: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 5947/52716 objects misplaced (11.281%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:28:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:28 smithi001 ceph-mon[112165]: pgmap v1512: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 5947/52716 objects misplaced (11.281%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:28:28.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.022 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:28 smithi012 ceph-mon[101180]: pgmap v1512: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 5947/52716 objects misplaced (11.281%); 14 MiB/s, 224 objects/s recovering 2024-09-14T04:28:29.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:29.149 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:29.251 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:29.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:29.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:29.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:29.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:29.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:29.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:29.654 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:29.699 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:28:29.699 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:28:29.711 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:29.724 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:28:29.724 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:28:29.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:28:29.752 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:28:29.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:29.781 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:28:29.782 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:28:29.789 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:29.806 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:28:29.807 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:28:29.838 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:28:29.838 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:28:29.871 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:28:29.871 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:28:29.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:29.901 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:28:29.902 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:28:29.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:29 smithi001 ceph-mon[107399]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:28:29.916 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:29 smithi001 ceph-mon[112165]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:28:29.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:29.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:29.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:29.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:28:30.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:30.071 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:29 smithi012 ceph-mon[101180]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:28:30.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.202 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:30.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:30.303 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:30.330 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:30.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:30.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:30.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:30.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:30.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:30.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:30.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.852 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:30.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:30.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:30.964 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[107399]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 970 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3973/46626 objects misplaced (8.521%); 12 MiB/s, 189 objects/s recovering 2024-09-14T04:28:30.965 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[107399]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:28:30.965 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4043814677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:30.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[112165]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 970 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3973/46626 objects misplaced (8.521%); 12 MiB/s, 189 objects/s recovering 2024-09-14T04:28:30.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[112165]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:28:30.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4043814677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:30.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:30.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:31.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:30 smithi012 ceph-mon[101180]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 112 active+clean; 970 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3973/46626 objects misplaced (8.521%); 12 MiB/s, 189 objects/s recovering 2024-09-14T04:28:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:30 smithi012 ceph-mon[101180]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:28:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4043814677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:31.116 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:31.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:31.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.244 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:31.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:31.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:31.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:31.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:31.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:31.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:31.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:31.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:31.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:31.759 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:31.860 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:31.893 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:31.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:31 smithi001 ceph-mon[112165]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:28:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:31 smithi001 ceph-mon[107399]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:28:31.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:32.004 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 7, 6] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:28:32.004 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:28:32.004 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2024-09-14T04:28:32.005 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:28:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.028 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:32.068 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:32.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:31 smithi012 ceph-mon[101180]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:28:32.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:32.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:32.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:32.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:32.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:32.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:32.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:32.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:32.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:32.672 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:32.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:32.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:32.950 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:32.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:32.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:32 smithi001 ceph-mon[112165]: pgmap v1517: 119 pgs: 1 peering, 5 active+remapped+backfill_wait, 113 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 3021/38817 objects misplaced (7.783%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:32.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:32 smithi001 ceph-mon[112165]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:28:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:32 smithi001 ceph-mon[107399]: pgmap v1517: 119 pgs: 1 peering, 5 active+remapped+backfill_wait, 113 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 3021/38817 objects misplaced (7.783%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:32 smithi001 ceph-mon[107399]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:28:32.999 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:28:33.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:33.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:32 smithi012 ceph-mon[101180]: pgmap v1517: 119 pgs: 1 peering, 5 active+remapped+backfill_wait, 113 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 3021/38817 objects misplaced (7.783%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:32 smithi012 ceph-mon[101180]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:28:33.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:33.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:33.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:33.399 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:33.500 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:33.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:33.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:33.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:33.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:33.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:33.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:33.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:33.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:33.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:33.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:33.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:33 smithi001 ceph-mon[107399]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:28:33.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:33 smithi001 ceph-mon[107399]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 113 active+clean; 775 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2479/37275 objects misplaced (6.651%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:33.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:33 smithi001 ceph-mon[112165]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:28:33.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:33 smithi001 ceph-mon[112165]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 113 active+clean; 775 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2479/37275 objects misplaced (6.651%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:34.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:34.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:33 smithi012 ceph-mon[101180]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:28:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:33 smithi012 ceph-mon[101180]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 peering, 3 active+remapped+backfill_wait, 113 active+clean; 775 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2479/37275 objects misplaced (6.651%); 13 MiB/s, 208 objects/s recovering 2024-09-14T04:28:34.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:34.167 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:34.268 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.292 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:34.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:34.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:34.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:34.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:34.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:34.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:34.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:34.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:34.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:34.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:34.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:34.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:34.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:34.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:34.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:34 smithi001 ceph-mon[107399]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:28:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:34 smithi001 ceph-mon[112165]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:28:35.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:35.054 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:35.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:34 smithi012 ceph-mon[101180]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:28:35.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:35.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:35.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:35.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:35.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:35.478 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:35.536 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:28:35.536 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:28:35.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:35.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.621 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:35.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:35.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.764 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:35.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:35.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:35.905 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:35.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[107399]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[107399]: pgmap v1523: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 686 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 328/32982 objects misplaced (0.994%); 25 MiB/s, 399 objects/s recovering 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[112165]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:28:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:35.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:35.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[112165]: pgmap v1523: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 686 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 328/32982 objects misplaced (0.994%); 25 MiB/s, 399 objects/s recovering 2024-09-14T04:28:35.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-14T04:28:36.006 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:36.038 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:35 smithi012 ceph-mon[101180]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:28:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:35 smithi012 ceph-mon[101180]: pgmap v1523: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 686 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 328/32982 objects misplaced (0.994%); 25 MiB/s, 399 objects/s recovering 2024-09-14T04:28:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-14T04:28:36.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:36.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:36.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:36.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:36.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:36.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:36.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:36.660 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:36.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:36.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:36.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:36.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:36.964 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:36] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:28:37.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:37.057 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:37.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:37.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:37.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-14T04:28:37.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:36 smithi001 ceph-mon[112165]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:28:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-14T04:28:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:36 smithi001 ceph-mon[107399]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:28:37.283 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:37.306 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:37.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-14T04:28:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:36 smithi012 ceph-mon[101180]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:28:37.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:37.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:37.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:37.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:37.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:37.582 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:37.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:37.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:37.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:37.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:37.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:37.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:37.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:37.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:38.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:38.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:38.124 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:38.156 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:28:38.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:38.207 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:37 smithi012 ceph-mon[101180]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:28:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:37 smithi012 ceph-mon[101180]: pgmap v1526: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 658 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 328/31623 objects misplaced (1.037%); 19 MiB/s, 306 objects/s recovering 2024-09-14T04:28:38.247 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:37 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-14T04:28:38.276 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[107399]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:28:38.276 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[107399]: pgmap v1526: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 658 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 328/31623 objects misplaced (1.037%); 19 MiB/s, 306 objects/s recovering 2024-09-14T04:28:38.276 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-14T04:28:38.276 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[112165]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:28:38.276 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[112165]: pgmap v1526: 119 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 116 active+clean; 658 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 328/31623 objects misplaced (1.037%); 19 MiB/s, 306 objects/s recovering 2024-09-14T04:28:38.277 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:37 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-14T04:28:38.309 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:38.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:38.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:38.465 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:38.567 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:38.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:38.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:38.693 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:38.716 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:38.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:38.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:38.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:38.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:38.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:38.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:38.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:38.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:39.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:39.090 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:39.192 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:39.214 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:39.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-14T04:28:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[107399]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:28:39.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[107399]: 13.15 starting backfill to osd.3 from (0'0,0'0] MAX to 472'11804 2024-09-14T04:28:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-14T04:28:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[112165]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:28:39.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:38 smithi001 ceph-mon[112165]: 13.15 starting backfill to osd.3 from (0'0,0'0] MAX to 472'11804 2024-09-14T04:28:39.317 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:39.339 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:38 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-14T04:28:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:38 smithi012 ceph-mon[101180]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:28:39.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:38 smithi012 ceph-mon[101180]: 13.15 starting backfill to osd.3 from (0'0,0'0] MAX to 472'11804 2024-09-14T04:28:39.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:39.464 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:39.568 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:39.591 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:39.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:39.717 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:39.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:39.842 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:39.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:39.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:39.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:39.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:40.001 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:28:40.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:40.093 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:40.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:40.230 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:40.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:40.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:39 smithi012 ceph-mon[101180]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:28:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1705671775' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:39 smithi012 ceph-mon[101180]: pgmap v1529: 119 pgs: 119 active+clean; 609 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 5.0 MiB/s, 79 objects/s recovering 2024-09-14T04:28:40.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:39 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-14T04:28:40.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:40.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:40.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:40.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[112165]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:28:40.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1705671775' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[112165]: pgmap v1529: 119 pgs: 119 active+clean; 609 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 5.0 MiB/s, 79 objects/s recovering 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[107399]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1705671775' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[107399]: pgmap v1529: 119 pgs: 119 active+clean; 609 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 5.0 MiB/s, 79 objects/s recovering 2024-09-14T04:28:40.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:39 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-14T04:28:40.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:40.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:40.522 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:40.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:40.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:40.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:40.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:40.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:40.843 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:40.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:40.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:40.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:40.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:40.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:41.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:41.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:41.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:41.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:41.241 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:41.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:40 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:28:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-14T04:28:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:40 smithi012 ceph-mon[101180]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:28:41.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:40 smithi012 ceph-mon[101180]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 473'11645 2024-09-14T04:28:41.366 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:41.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:28:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-14T04:28:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[112165]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:28:41.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[112165]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 473'11645 2024-09-14T04:28:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:28:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-14T04:28:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[107399]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:28:41.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:40 smithi001 ceph-mon[107399]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 473'11645 2024-09-14T04:28:41.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:41.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:41.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.616 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:41.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:41.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:41.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:41.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:41.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:41.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:41.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:41.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:41.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:42.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:42.113 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[107399]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:28:42.113 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[107399]: pgmap v1532: 119 pgs: 119 active+clean; 546 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 8.9 MiB/s, 142 objects/s recovering 2024-09-14T04:28:42.113 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-14T04:28:42.114 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[112165]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:28:42.114 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[112165]: pgmap v1532: 119 pgs: 119 active+clean; 546 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 8.9 MiB/s, 142 objects/s recovering 2024-09-14T04:28:42.114 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:42 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-14T04:28:42.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:42.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:42.261 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:42 smithi012 ceph-mon[101180]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:28:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:42 smithi012 ceph-mon[101180]: pgmap v1532: 119 pgs: 119 active+clean; 546 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 8.9 MiB/s, 142 objects/s recovering 2024-09-14T04:28:42.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:42 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-14T04:28:42.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:42.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:42.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:42.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:42.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:42.671 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:42.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:42.801 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:42.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:42.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:42.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:43.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:43.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:43.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:43.134 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:28:43.135 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:28:43.152 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:28:43.153 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:28:43.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:43.170 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:28:43.170 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:28:43.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:43.187 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:28:43.188 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:28:43.205 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:28:43.205 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:28:43.223 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:28:43.223 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:28:43.242 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:28:43.242 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:28:43.259 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:28:43.259 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[112165]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[112165]: 13.17 starting backfill to osd.0 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[112165]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 475'11728 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[112165]: 13.17 starting backfill to osd.5 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[107399]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[107399]: 13.17 starting backfill to osd.0 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[107399]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 475'11728 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:43 smithi001 ceph-mon[107399]: 13.17 starting backfill to osd.5 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:43.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:43 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-14T04:28:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:43 smithi012 ceph-mon[101180]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:28:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:43 smithi012 ceph-mon[101180]: 13.17 starting backfill to osd.0 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:43 smithi012 ceph-mon[101180]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 475'11728 2024-09-14T04:28:43.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:43 smithi012 ceph-mon[101180]: 13.17 starting backfill to osd.5 from (420'4100,466'11587] MIN to 475'11728 2024-09-14T04:28:43.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:43.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:43.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:43.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:43.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:43.661 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.684 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:43.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:43.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:43.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:43.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:43.933 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:43.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:44.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:44.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:44.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.196 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:44.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:44.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[107399]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4165191834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[107399]: pgmap v1535: 119 pgs: 1 peering, 118 active+clean; 530 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 315 op/s; 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[107399]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[112165]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4165191834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[112165]: pgmap v1535: 119 pgs: 1 peering, 118 active+clean; 530 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 315 op/s; 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:44.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:44 smithi001 ceph-mon[112165]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:28:44.297 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:44.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:44.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:44 smithi012 ceph-mon[101180]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:28:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:44 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4165191834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:44 smithi012 ceph-mon[101180]: pgmap v1535: 119 pgs: 1 peering, 118 active+clean; 530 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 315 op/s; 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:28:44.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:44 smithi012 ceph-mon[101180]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:28:44.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:44.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:44.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:44.575 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:44.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:44.709 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:44.734 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:44.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:44.836 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:44.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:44.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:44.983 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:44.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:45.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:45.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:45.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:45.211 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:45.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[112165]: 13.18 starting backfill to osd.0 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[112165]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[112165]: 13.18 starting backfill to osd.7 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[107399]: 13.18 starting backfill to osd.0 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[107399]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:45 smithi001 ceph-mon[107399]: 13.18 starting backfill to osd.7 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.247 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:45 smithi012 ceph-mon[101180]: 13.18 starting backfill to osd.0 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:45 smithi012 ceph-mon[101180]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:45 smithi012 ceph-mon[101180]: 13.18 starting backfill to osd.7 from (0'0,0'0] MAX to 477'11479 2024-09-14T04:28:45.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:45.378 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:45.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:45.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:45.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:28:45.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:45.646 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:45.734 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:28:45.734 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:28:45.734 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:28:45.734 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:28:45.734 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:28:45.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:45.771 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:45.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:45.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:45.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:45.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:45.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:46.021 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:46.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:46.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:46.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:46.302 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:46 smithi012 ceph-mon[101180]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:28:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:46 smithi012 ceph-mon[101180]: pgmap v1538: 119 pgs: 2 peering, 117 active+clean; 464 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-09-14T04:28:46.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:46 smithi012 ceph-mon[101180]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:28:46.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:46.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[112165]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[112165]: pgmap v1538: 119 pgs: 2 peering, 117 active+clean; 464 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[112165]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[107399]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[107399]: pgmap v1538: 119 pgs: 2 peering, 117 active+clean; 464 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-09-14T04:28:46.441 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-mon[107399]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:28:46.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:28:46.542 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:46.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:46.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:46.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.766 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:46.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:46.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:46.894 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:46.923 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:46.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:46] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-14T04:28:46.973 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:46.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:28:47.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:47.058 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:47.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:47.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:47.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:47.205 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:28:47.207 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:28:47.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:47.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:47.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:47.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:47.484 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:47.586 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:47.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:47.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:47.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:47.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:47.919 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:47.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:48.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:48.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:48.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:48.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:48.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:48.456 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:48.495 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:48.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:48.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.639 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:48.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:48.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:48.776 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:48.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:48.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:48.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:48.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:48 smithi001 ceph-mon[107399]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:28:48.929 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:48 smithi001 ceph-mon[107399]: pgmap v1540: 119 pgs: 1 peering, 118 active+clean; 431 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 6.1 MiB/s, 96 objects/s recovering 2024-09-14T04:28:48.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:48 smithi001 ceph-mon[112165]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:28:48.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:48 smithi001 ceph-mon[112165]: pgmap v1540: 119 pgs: 1 peering, 118 active+clean; 431 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 6.1 MiB/s, 96 objects/s recovering 2024-09-14T04:28:49.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:49.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:48 smithi012 ceph-mon[101180]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:28:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:48 smithi012 ceph-mon[101180]: pgmap v1540: 119 pgs: 1 peering, 118 active+clean; 431 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 6.1 MiB/s, 96 objects/s recovering 2024-09-14T04:28:49.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:49.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:49.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:49.340 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:49.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:49.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:49.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:49.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:49.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:49.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:49.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.757 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:49.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:49.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:49.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:49.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:50.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.047 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:50.096 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1539,"stamp":"2024-09-14T04:28:46.018254+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":486742013,"num_objects":7445,"num_object_clones":0,"num_object_copies":22335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7445,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134972,"num_write_kb":2843938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22804,"num_bytes_recovered":1491557370,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":204074,"ondisk_log_size":204074,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":381,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9384432,"kb_used_data":1801156,"kb_used_omap":41,"kb_used_meta":7582998,"kb_avail":740576784,"statfs":{"total":767960285184,"available":758350626816,"internally_reserved":0,"allocated":1844383744,"data_stored":1819352611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42935,"internal_metadata":7764990025},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-80936960,"num_objects":-1235,"num_object_clones":0,"num_object_copies":-3705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1235,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":216,"num_bytes_recovered":14155776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.011982"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":766,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010171+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:28:44.010171+0000","last_peered":"2024-09-14T04:28:44.010171+0000","last_clean":"2024-09-14T04:28:44.010171+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:28:44.010171+0000","last_undegraded":"2024-09-14T04:28:44.010171+0000","last_fullsized":"2024-09-14T04:28:44.010171+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":277,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015243+0000","last_change":"2024-09-14T04:27:20.240919+0000","last_active":"2024-09-14T04:28:43.015243+0000","last_peered":"2024-09-14T04:28:43.015243+0000","last_clean":"2024-09-14T04:28:43.015243+0000","last_became_active":"2024-09-14T04:27:20.240514+0000","last_became_peered":"2024-09-14T04:27:20.240514+0000","last_unstale":"2024-09-14T04:28:43.015243+0000","last_undegraded":"2024-09-14T04:28:43.015243+0000","last_fullsized":"2024-09-14T04:28:43.015243+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:55.818976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":57,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010520+0000","last_change":"2024-09-14T04:27:20.354646+0000","last_active":"2024-09-14T04:28:44.010520+0000","last_peered":"2024-09-14T04:28:44.010520+0000","last_clean":"2024-09-14T04:28:44.010520+0000","last_became_active":"2024-09-14T04:27:20.242671+0000","last_became_peered":"2024-09-14T04:27:20.242671+0000","last_unstale":"2024-09-14T04:28:44.010520+0000","last_undegraded":"2024-09-14T04:28:44.010520+0000","last_fullsized":"2024-09-14T04:28:44.010520+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:14.740647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":834,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999440+0000","last_change":"2024-09-14T04:27:23.243063+0000","last_active":"2024-09-14T04:28:39.999440+0000","last_peered":"2024-09-14T04:28:39.999440+0000","last_clean":"2024-09-14T04:28:39.999440+0000","last_became_active":"2024-09-14T04:27:23.242547+0000","last_became_peered":"2024-09-14T04:27:23.242547+0000","last_unstale":"2024-09-14T04:28:39.999440+0000","last_undegraded":"2024-09-14T04:28:39.999440+0000","last_fullsized":"2024-09-14T04:28:39.999440+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:29.267328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.000357+0000","last_change":"2024-09-14T04:27:20.243359+0000","last_active":"2024-09-14T04:28:40.000357+0000","last_peered":"2024-09-14T04:28:40.000357+0000","last_clean":"2024-09-14T04:28:40.000357+0000","last_became_active":"2024-09-14T04:27:20.243021+0000","last_became_peered":"2024-09-14T04:27:20.243021+0000","last_unstale":"2024-09-14T04:28:40.000357+0000","last_undegraded":"2024-09-14T04:28:40.000357+0000","last_fullsized":"2024-09-14T04:28:40.000357+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:58:10.694816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":589,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010292+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:28:44.010292+0000","last_peered":"2024-09-14T04:28:44.010292+0000","last_clean":"2024-09-14T04:28:44.010292+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:28:44.010292+0000","last_undegraded":"2024-09-14T04:28:44.010292+0000","last_fullsized":"2024-09-14T04:28:44.010292+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":1054,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014971+0000","last_change":"2024-09-14T04:25:38.620883+0000","last_active":"2024-09-14T04:28:43.014971+0000","last_peered":"2024-09-14T04:28:43.014971+0000","last_clean":"2024-09-14T04:28:43.014971+0000","last_became_active":"2024-09-14T04:25:38.620413+0000","last_became_peered":"2024-09-14T04:25:38.620413+0000","last_unstale":"2024-09-14T04:28:43.014971+0000","last_undegraded":"2024-09-14T04:28:43.014971+0000","last_fullsized":"2024-09-14T04:28:43.014971+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:01.846654+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":575,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010731+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:28:44.010731+0000","last_peered":"2024-09-14T04:28:44.010731+0000","last_clean":"2024-09-14T04:28:44.010731+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:28:44.010731+0000","last_undegraded":"2024-09-14T04:28:44.010731+0000","last_fullsized":"2024-09-14T04:28:44.010731+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010554+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:28:44.010554+0000","last_peered":"2024-09-14T04:28:44.010554+0000","last_clean":"2024-09-14T04:28:44.010554+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:28:44.010554+0000","last_undegraded":"2024-09-14T04:28:44.010554+0000","last_fullsized":"2024-09-14T04:28:44.010554+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":679,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999018+0000","last_change":"2024-09-14T04:28:22.704112+0000","last_active":"2024-09-14T04:28:39.999018+0000","last_peered":"2024-09-14T04:28:39.999018+0000","last_clean":"2024-09-14T04:28:39.999018+0000","last_became_active":"2024-09-14T04:28:22.703746+0000","last_became_peered":"2024-09-14T04:28:22.703746+0000","last_unstale":"2024-09-14T04:28:39.999018+0000","last_undegraded":"2024-09-14T04:28:39.999018+0000","last_fullsized":"2024-09-14T04:28:39.999018+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:28.446771+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":148,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.005260+0000","last_change":"2024-09-14T04:27:20.236945+0000","last_active":"2024-09-14T04:28:41.005260+0000","last_peered":"2024-09-14T04:28:41.005260+0000","last_clean":"2024-09-14T04:28:41.005260+0000","last_became_active":"2024-09-14T04:27:20.236600+0000","last_became_peered":"2024-09-14T04:27:20.236600+0000","last_unstale":"2024-09-14T04:28:41.005260+0000","last_undegraded":"2024-09-14T04:28:41.005260+0000","last_fullsized":"2024-09-14T04:28:41.005260+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:49.622182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":42,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010288+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:28:44.010288+0000","last_peered":"2024-09-14T04:28:44.010288+0000","last_clean":"2024-09-14T04:28:44.010288+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:28:44.010288+0000","last_undegraded":"2024-09-14T04:28:44.010288+0000","last_fullsized":"2024-09-14T04:28:44.010288+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"479'11352","reported_seq":14666,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.557305+0000","last_change":"2024-09-14T04:27:55.267835+0000","last_active":"2024-09-14T04:28:42.557305+0000","last_peered":"2024-09-14T04:28:42.557305+0000","last_clean":"2024-09-14T04:28:42.557305+0000","last_became_active":"2024-09-14T04:27:55.255201+0000","last_became_peered":"2024-09-14T04:27:55.255201+0000","last_unstale":"2024-09-14T04:28:42.557305+0000","last_undegraded":"2024-09-14T04:28:42.557305+0000","last_fullsized":"2024-09-14T04:28:42.557305+0000","mapping_epoch":452,"log_start":"409'2200","ondisk_log_start":"409'2200","created":398,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'8948","last_scrub_stamp":"2024-09-14T04:27:07.957405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.957405+0000","objects_scrubbed":2669,"log_size":9152,"log_dups_size":0,"ondisk_log_size":9152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:45.422896+0000","scrub_duration":925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121856,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1535,"num_bytes_recovered":100597760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":750,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011276+0000","last_change":"2024-09-14T04:27:23.241978+0000","last_active":"2024-09-14T04:28:44.011276+0000","last_peered":"2024-09-14T04:28:44.011276+0000","last_clean":"2024-09-14T04:28:44.011276+0000","last_became_active":"2024-09-14T04:27:23.241538+0000","last_became_peered":"2024-09-14T04:27:23.241538+0000","last_unstale":"2024-09-14T04:28:44.011276+0000","last_undegraded":"2024-09-14T04:28:44.011276+0000","last_fullsized":"2024-09-14T04:28:44.011276+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:51:14.548348+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":55,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011204+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:28:44.011204+0000","last_peered":"2024-09-14T04:28:44.011204+0000","last_clean":"2024-09-14T04:28:44.011204+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:28:44.011204+0000","last_undegraded":"2024-09-14T04:28:44.011204+0000","last_fullsized":"2024-09-14T04:28:44.011204+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999169+0000","last_change":"2024-09-14T04:27:20.241919+0000","last_active":"2024-09-14T04:28:39.999169+0000","last_peered":"2024-09-14T04:28:39.999169+0000","last_clean":"2024-09-14T04:28:39.999169+0000","last_became_active":"2024-09-14T04:27:20.241583+0000","last_became_peered":"2024-09-14T04:27:20.241583+0000","last_unstale":"2024-09-14T04:28:39.999169+0000","last_undegraded":"2024-09-14T04:28:39.999169+0000","last_fullsized":"2024-09-14T04:28:39.999169+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:00:28.708616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"481'11734","reported_seq":15475,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.465023+0000","last_change":"2024-09-14T04:27:47.660951+0000","last_active":"2024-09-14T04:28:44.465023+0000","last_peered":"2024-09-14T04:28:44.465023+0000","last_clean":"2024-09-14T04:28:44.465023+0000","last_became_active":"2024-09-14T04:27:47.640210+0000","last_became_peered":"2024-09-14T04:27:47.640210+0000","last_unstale":"2024-09-14T04:28:44.465023+0000","last_undegraded":"2024-09-14T04:28:44.465023+0000","last_fullsized":"2024-09-14T04:28:44.465023+0000","mapping_epoch":450,"log_start":"420'3900","ondisk_log_start":"420'3900","created":398,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9193","last_scrub_stamp":"2024-09-14T04:27:06.120770+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.120770+0000","objects_scrubbed":2771,"log_size":7834,"log_dups_size":0,"ondisk_log_size":7834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:10.142725+0000","scrub_duration":1072,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2557,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1765,"num_bytes_recovered":115671040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1117,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.005159+0000","last_change":"2024-09-14T04:28:22.698300+0000","last_active":"2024-09-14T04:28:41.005159+0000","last_peered":"2024-09-14T04:28:41.005159+0000","last_clean":"2024-09-14T04:28:41.005159+0000","last_became_active":"2024-09-14T04:28:22.697894+0000","last_became_peered":"2024-09-14T04:28:22.697894+0000","last_unstale":"2024-09-14T04:28:41.005159+0000","last_undegraded":"2024-09-14T04:28:41.005159+0000","last_fullsized":"2024-09-14T04:28:41.005159+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:19.688955+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":55,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011227+0000","last_change":"2024-09-14T04:27:20.256429+0000","last_active":"2024-09-14T04:28:44.011227+0000","last_peered":"2024-09-14T04:28:44.011227+0000","last_clean":"2024-09-14T04:28:44.011227+0000","last_became_active":"2024-09-14T04:27:20.238968+0000","last_became_peered":"2024-09-14T04:27:20.238968+0000","last_unstale":"2024-09-14T04:28:44.011227+0000","last_undegraded":"2024-09-14T04:28:44.011227+0000","last_fullsized":"2024-09-14T04:28:44.011227+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:19.325270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":509,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014845+0000","last_change":"2024-09-14T04:25:35.561729+0000","last_active":"2024-09-14T04:28:44.014845+0000","last_peered":"2024-09-14T04:28:44.014845+0000","last_clean":"2024-09-14T04:28:44.014845+0000","last_became_active":"2024-09-14T04:25:35.561510+0000","last_became_peered":"2024-09-14T04:25:35.561510+0000","last_unstale":"2024-09-14T04:28:44.014845+0000","last_undegraded":"2024-09-14T04:28:44.014845+0000","last_fullsized":"2024-09-14T04:28:44.014845+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:20:51.441207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"481'11782","reported_seq":12015,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.531474+0000","last_change":"2024-09-14T04:27:06.468498+0000","last_active":"2024-09-14T04:28:44.531474+0000","last_peered":"2024-09-14T04:28:44.531474+0000","last_clean":"2024-09-14T04:28:44.531474+0000","last_became_active":"2024-09-14T04:27:04.629985+0000","last_became_peered":"2024-09-14T04:27:04.629985+0000","last_unstale":"2024-09-14T04:28:44.531474+0000","last_undegraded":"2024-09-14T04:28:44.531474+0000","last_fullsized":"2024-09-14T04:28:44.531474+0000","mapping_epoch":398,"log_start":"418'3600","ondisk_log_start":"418'3600","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9192","last_scrub_stamp":"2024-09-14T04:27:06.468321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.468321+0000","objects_scrubbed":2840,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:27:15.398278+0000","scrub_duration":864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15073280,"num_objects":230,"num_object_clones":0,"num_object_copies":690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2610,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":431,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010958+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:28:44.010958+0000","last_peered":"2024-09-14T04:28:44.010958+0000","last_clean":"2024-09-14T04:28:44.010958+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:28:44.010958+0000","last_undegraded":"2024-09-14T04:28:44.010958+0000","last_fullsized":"2024-09-14T04:28:44.010958+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":68,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011150+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:28:44.011150+0000","last_peered":"2024-09-14T04:28:44.011150+0000","last_clean":"2024-09-14T04:28:44.011150+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:28:44.011150+0000","last_undegraded":"2024-09-14T04:28:44.011150+0000","last_fullsized":"2024-09-14T04:28:44.011150+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":560,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015095+0000","last_change":"2024-09-14T04:27:20.255490+0000","last_active":"2024-09-14T04:28:43.015095+0000","last_peered":"2024-09-14T04:28:43.015095+0000","last_clean":"2024-09-14T04:28:43.015095+0000","last_became_active":"2024-09-14T04:27:20.255200+0000","last_became_peered":"2024-09-14T04:27:20.255200+0000","last_unstale":"2024-09-14T04:28:43.015095+0000","last_undegraded":"2024-09-14T04:28:43.015095+0000","last_fullsized":"2024-09-14T04:28:43.015095+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:06.262888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"477'11681","reported_seq":18063,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.577871+0000","last_change":"2024-09-14T04:27:08.799372+0000","last_active":"2024-09-14T04:28:40.577871+0000","last_peered":"2024-09-14T04:28:40.577871+0000","last_clean":"2024-09-14T04:28:40.577871+0000","last_became_active":"2024-09-14T04:27:04.633656+0000","last_became_peered":"2024-09-14T04:27:04.633656+0000","last_unstale":"2024-09-14T04:28:40.577871+0000","last_undegraded":"2024-09-14T04:28:40.577871+0000","last_fullsized":"2024-09-14T04:28:40.577871+0000","mapping_epoch":425,"log_start":"420'3845","ondisk_log_start":"420'3845","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'9212","last_scrub_stamp":"2024-09-14T04:27:08.799175+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:08.799175+0000","objects_scrubbed":2719,"log_size":7836,"log_dups_size":0,"ondisk_log_size":7836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:13.865312+0000","scrub_duration":0.64896177700000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15859712,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2477,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":488,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.013615+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:28:44.013615+0000","last_peered":"2024-09-14T04:28:44.013615+0000","last_clean":"2024-09-14T04:28:44.013615+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:28:44.013615+0000","last_undegraded":"2024-09-14T04:28:44.013615+0000","last_fullsized":"2024-09-14T04:28:44.013615+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":437,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.001299+0000","last_change":"2024-09-14T04:28:19.259159+0000","last_active":"2024-09-14T04:28:40.001299+0000","last_peered":"2024-09-14T04:28:40.001299+0000","last_clean":"2024-09-14T04:28:40.001299+0000","last_became_active":"2024-09-14T04:28:19.258744+0000","last_became_peered":"2024-09-14T04:28:19.258744+0000","last_unstale":"2024-09-14T04:28:40.001299+0000","last_undegraded":"2024-09-14T04:28:40.001299+0000","last_fullsized":"2024-09-14T04:28:40.001299+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:13.134482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":562,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010885+0000","last_change":"2024-09-14T04:10:35.385486+0000","last_active":"2024-09-14T04:28:44.010885+0000","last_peered":"2024-09-14T04:28:44.010885+0000","last_clean":"2024-09-14T04:28:44.010885+0000","last_became_active":"2024-09-14T04:10:35.383336+0000","last_became_peered":"2024-09-14T04:10:35.383336+0000","last_unstale":"2024-09-14T04:28:44.010885+0000","last_undegraded":"2024-09-14T04:28:44.010885+0000","last_fullsized":"2024-09-14T04:28:44.010885+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:05.809194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"482'11845","reported_seq":12013,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.246493+0000","last_change":"2024-09-14T04:27:09.662483+0000","last_active":"2024-09-14T04:28:45.246493+0000","last_peered":"2024-09-14T04:28:45.246493+0000","last_clean":"2024-09-14T04:28:45.246493+0000","last_became_active":"2024-09-14T04:27:04.651733+0000","last_became_peered":"2024-09-14T04:27:04.651733+0000","last_unstale":"2024-09-14T04:28:45.246493+0000","last_undegraded":"2024-09-14T04:28:45.246493+0000","last_fullsized":"2024-09-14T04:28:45.246493+0000","mapping_epoch":398,"log_start":"420'4100","ondisk_log_start":"420'4100","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"432'9425","last_scrub_stamp":"2024-09-14T04:27:09.662329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:09.662329+0000","objects_scrubbed":2639,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:50:00.812272+0000","scrub_duration":0.53680546200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14090240,"num_objects":215,"num_object_clones":0,"num_object_copies":645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2424,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":817,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010645+0000","last_change":"2024-09-14T04:27:23.258324+0000","last_active":"2024-09-14T04:28:44.010645+0000","last_peered":"2024-09-14T04:28:44.010645+0000","last_clean":"2024-09-14T04:28:44.010645+0000","last_became_active":"2024-09-14T04:27:23.257916+0000","last_became_peered":"2024-09-14T04:27:23.257916+0000","last_unstale":"2024-09-14T04:28:44.010645+0000","last_undegraded":"2024-09-14T04:28:44.010645+0000","last_fullsized":"2024-09-14T04:28:44.010645+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:29:08.763999+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":566,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010488+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:28:44.010488+0000","last_peered":"2024-09-14T04:28:44.010488+0000","last_clean":"2024-09-14T04:28:44.010488+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:28:44.010488+0000","last_undegraded":"2024-09-14T04:28:44.010488+0000","last_fullsized":"2024-09-14T04:28:44.010488+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":484,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.013936+0000","last_change":"2024-09-14T04:14:43.299941+0000","last_active":"2024-09-14T04:28:44.013936+0000","last_peered":"2024-09-14T04:28:44.013936+0000","last_clean":"2024-09-14T04:28:44.013936+0000","last_became_active":"2024-09-14T04:14:43.299226+0000","last_became_peered":"2024-09-14T04:14:43.299226+0000","last_unstale":"2024-09-14T04:28:44.013936+0000","last_undegraded":"2024-09-14T04:28:44.013936+0000","last_fullsized":"2024-09-14T04:28:44.013936+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:55:01.710493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"479'11877","reported_seq":12105,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.572363+0000","last_change":"2024-09-14T04:27:07.089728+0000","last_active":"2024-09-14T04:28:42.572363+0000","last_peered":"2024-09-14T04:28:42.572363+0000","last_clean":"2024-09-14T04:28:42.572363+0000","last_became_active":"2024-09-14T04:27:04.639444+0000","last_became_peered":"2024-09-14T04:27:04.639444+0000","last_unstale":"2024-09-14T04:28:42.572363+0000","last_undegraded":"2024-09-14T04:28:42.572363+0000","last_fullsized":"2024-09-14T04:28:42.572363+0000","mapping_epoch":398,"log_start":"412'2700","ondisk_log_start":"412'2700","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9334","last_scrub_stamp":"2024-09-14T04:27:07.089071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.089071+0000","objects_scrubbed":2819,"log_size":9177,"log_dups_size":0,"ondisk_log_size":9177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:10.818373+0000","scrub_duration":1039,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17039360,"num_objects":260,"num_object_clones":0,"num_object_copies":780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2559,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":713,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999261+0000","last_change":"2024-09-14T04:25:28.614151+0000","last_active":"2024-09-14T04:28:39.999261+0000","last_peered":"2024-09-14T04:28:39.999261+0000","last_clean":"2024-09-14T04:28:39.999261+0000","last_became_active":"2024-09-14T04:25:28.613891+0000","last_became_peered":"2024-09-14T04:25:28.613891+0000","last_unstale":"2024-09-14T04:28:39.999261+0000","last_undegraded":"2024-09-14T04:28:39.999261+0000","last_fullsized":"2024-09-14T04:28:39.999261+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:46.049408+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":32,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010085+0000","last_change":"2024-09-14T04:28:19.260945+0000","last_active":"2024-09-14T04:28:44.010085+0000","last_peered":"2024-09-14T04:28:44.010085+0000","last_clean":"2024-09-14T04:28:44.010085+0000","last_became_active":"2024-09-14T04:28:19.260647+0000","last_became_peered":"2024-09-14T04:28:19.260647+0000","last_unstale":"2024-09-14T04:28:44.010085+0000","last_undegraded":"2024-09-14T04:28:44.010085+0000","last_fullsized":"2024-09-14T04:28:44.010085+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:06:26.758129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014757+0000","last_change":"2024-09-14T04:25:07.967431+0000","last_active":"2024-09-14T04:28:44.014757+0000","last_peered":"2024-09-14T04:28:44.014757+0000","last_clean":"2024-09-14T04:28:44.014757+0000","last_became_active":"2024-09-14T04:25:07.967052+0000","last_became_peered":"2024-09-14T04:25:07.967052+0000","last_unstale":"2024-09-14T04:28:44.014757+0000","last_undegraded":"2024-09-14T04:28:44.014757+0000","last_fullsized":"2024-09-14T04:28:44.014757+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:03.319597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"482'11857","reported_seq":19501,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.021379+0000","last_change":"2024-09-14T04:28:32.714654+0000","last_active":"2024-09-14T04:28:45.021379+0000","last_peered":"2024-09-14T04:28:45.021379+0000","last_clean":"2024-09-14T04:28:45.021379+0000","last_became_active":"2024-09-14T04:28:32.678741+0000","last_became_peered":"2024-09-14T04:28:32.678741+0000","last_unstale":"2024-09-14T04:28:45.021379+0000","last_undegraded":"2024-09-14T04:28:45.021379+0000","last_fullsized":"2024-09-14T04:28:45.021379+0000","mapping_epoch":469,"log_start":"420'4100","ondisk_log_start":"420'4100","created":398,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9272","last_scrub_stamp":"2024-09-14T04:27:06.713334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.713334+0000","objects_scrubbed":2843,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:32.899422+0000","scrub_duration":0.58335741600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15728640,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2603,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1420,"num_bytes_recovered":93061120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":1034,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014526+0000","last_change":"2024-09-14T04:24:49.575647+0000","last_active":"2024-09-14T04:28:43.014526+0000","last_peered":"2024-09-14T04:28:43.014526+0000","last_clean":"2024-09-14T04:28:43.014526+0000","last_became_active":"2024-09-14T04:24:49.575434+0000","last_became_peered":"2024-09-14T04:24:49.575434+0000","last_unstale":"2024-09-14T04:28:43.014526+0000","last_undegraded":"2024-09-14T04:28:43.014526+0000","last_fullsized":"2024-09-14T04:28:43.014526+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:15.585642+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":101,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014594+0000","last_change":"2024-09-14T04:27:20.244483+0000","last_active":"2024-09-14T04:28:44.014594+0000","last_peered":"2024-09-14T04:28:44.014594+0000","last_clean":"2024-09-14T04:28:44.014594+0000","last_became_active":"2024-09-14T04:27:20.244182+0000","last_became_peered":"2024-09-14T04:27:20.244182+0000","last_unstale":"2024-09-14T04:28:44.014594+0000","last_undegraded":"2024-09-14T04:28:44.014594+0000","last_fullsized":"2024-09-14T04:28:44.014594+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:36.658365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":490,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.015067+0000","last_change":"2024-09-14T04:23:35.660779+0000","last_active":"2024-09-14T04:28:44.015067+0000","last_peered":"2024-09-14T04:28:44.015067+0000","last_clean":"2024-09-14T04:28:44.015067+0000","last_became_active":"2024-09-14T04:23:35.660561+0000","last_became_peered":"2024-09-14T04:23:35.660561+0000","last_unstale":"2024-09-14T04:28:44.015067+0000","last_undegraded":"2024-09-14T04:28:44.015067+0000","last_fullsized":"2024-09-14T04:28:44.015067+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:48.996381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"482'11731","reported_seq":11902,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.229501+0000","last_change":"2024-09-14T04:27:05.659073+0000","last_active":"2024-09-14T04:28:45.229501+0000","last_peered":"2024-09-14T04:28:45.229501+0000","last_clean":"2024-09-14T04:28:45.229501+0000","last_became_active":"2024-09-14T04:27:04.643791+0000","last_became_peered":"2024-09-14T04:27:04.643791+0000","last_unstale":"2024-09-14T04:28:45.229501+0000","last_undegraded":"2024-09-14T04:28:45.229501+0000","last_fullsized":"2024-09-14T04:28:45.229501+0000","mapping_epoch":398,"log_start":"420'3900","ondisk_log_start":"420'3900","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9109","last_scrub_stamp":"2024-09-14T04:27:05.658918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:05.658918+0000","objects_scrubbed":2855,"log_size":7831,"log_dups_size":0,"ondisk_log_size":7831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:50.057418+0000","scrub_duration":0.52478999100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14417920,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2635,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":701,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999220+0000","last_change":"2024-09-14T04:28:22.703134+0000","last_active":"2024-09-14T04:28:39.999220+0000","last_peered":"2024-09-14T04:28:39.999220+0000","last_clean":"2024-09-14T04:28:39.999220+0000","last_became_active":"2024-09-14T04:28:22.702723+0000","last_became_peered":"2024-09-14T04:28:22.702723+0000","last_unstale":"2024-09-14T04:28:39.999220+0000","last_undegraded":"2024-09-14T04:28:39.999220+0000","last_fullsized":"2024-09-14T04:28:39.999220+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:45.568551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:45.568551+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:08:32.899634+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":148,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.004645+0000","last_change":"2024-09-14T04:27:20.236754+0000","last_active":"2024-09-14T04:28:41.004645+0000","last_peered":"2024-09-14T04:28:41.004645+0000","last_clean":"2024-09-14T04:28:41.004645+0000","last_became_active":"2024-09-14T04:27:20.236432+0000","last_became_peered":"2024-09-14T04:27:20.236432+0000","last_unstale":"2024-09-14T04:28:41.004645+0000","last_undegraded":"2024-09-14T04:28:41.004645+0000","last_fullsized":"2024-09-14T04:28:41.004645+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:43.774063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":803,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015296+0000","last_change":"2024-09-14T04:06:46.358757+0000","last_active":"2024-09-14T04:28:43.015296+0000","last_peered":"2024-09-14T04:28:43.015296+0000","last_clean":"2024-09-14T04:28:43.015296+0000","last_became_active":"2024-09-14T04:06:46.358362+0000","last_became_peered":"2024-09-14T04:06:46.358362+0000","last_unstale":"2024-09-14T04:28:43.015296+0000","last_undegraded":"2024-09-14T04:28:43.015296+0000","last_fullsized":"2024-09-14T04:28:43.015296+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:58.332088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"480'14594","reported_seq":14782,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.479953+0000","last_change":"2024-09-14T04:25:01.993531+0000","last_active":"2024-09-14T04:28:43.479953+0000","last_peered":"2024-09-14T04:28:43.479953+0000","last_clean":"2024-09-14T04:28:43.479953+0000","last_became_active":"2024-09-14T04:25:01.993257+0000","last_became_peered":"2024-09-14T04:25:01.993257+0000","last_unstale":"2024-09-14T04:28:43.479953+0000","last_undegraded":"2024-09-14T04:28:43.479953+0000","last_fullsized":"2024-09-14T04:28:43.479953+0000","mapping_epoch":398,"log_start":"428'8708","ondisk_log_start":"428'8708","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:30:21.097731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29753344,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14594,"num_write_kb":481536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":889,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010163+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:28:44.010163+0000","last_peered":"2024-09-14T04:28:44.010163+0000","last_clean":"2024-09-14T04:28:44.010163+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:28:44.010163+0000","last_undegraded":"2024-09-14T04:28:44.010163+0000","last_fullsized":"2024-09-14T04:28:44.010163+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":431,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.000852+0000","last_change":"2024-09-14T04:27:20.357956+0000","last_active":"2024-09-14T04:28:40.000852+0000","last_peered":"2024-09-14T04:28:40.000852+0000","last_clean":"2024-09-14T04:28:40.000852+0000","last_became_active":"2024-09-14T04:27:20.241613+0000","last_became_peered":"2024-09-14T04:27:20.241613+0000","last_unstale":"2024-09-14T04:28:40.000852+0000","last_undegraded":"2024-09-14T04:28:40.000852+0000","last_fullsized":"2024-09-14T04:28:40.000852+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:06.823240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":134,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.011795+0000","last_change":"2024-09-14T04:28:19.252275+0000","last_active":"2024-09-14T04:28:42.011795+0000","last_peered":"2024-09-14T04:28:42.011795+0000","last_clean":"2024-09-14T04:28:42.011795+0000","last_became_active":"2024-09-14T04:28:19.251529+0000","last_became_peered":"2024-09-14T04:28:19.251529+0000","last_unstale":"2024-09-14T04:28:42.011795+0000","last_undegraded":"2024-09-14T04:28:42.011795+0000","last_fullsized":"2024-09-14T04:28:42.011795+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:38.172466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014719+0000","last_change":"2024-09-14T04:27:20.247824+0000","last_active":"2024-09-14T04:28:44.014719+0000","last_peered":"2024-09-14T04:28:44.014719+0000","last_clean":"2024-09-14T04:28:44.014719+0000","last_became_active":"2024-09-14T04:27:20.247205+0000","last_became_peered":"2024-09-14T04:27:20.247205+0000","last_unstale":"2024-09-14T04:28:44.014719+0000","last_undegraded":"2024-09-14T04:28:44.014719+0000","last_fullsized":"2024-09-14T04:28:44.014719+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:09.902975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"482'14295","reported_seq":19135,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.148240+0000","last_change":"2024-09-14T04:28:29.726612+0000","last_active":"2024-09-14T04:28:45.148240+0000","last_peered":"2024-09-14T04:28:45.148240+0000","last_clean":"2024-09-14T04:28:45.148240+0000","last_became_active":"2024-09-14T04:28:29.670709+0000","last_became_peered":"2024-09-14T04:28:29.670709+0000","last_unstale":"2024-09-14T04:28:45.148240+0000","last_undegraded":"2024-09-14T04:28:45.148240+0000","last_fullsized":"2024-09-14T04:28:45.148240+0000","mapping_epoch":466,"log_start":"424'6550","ondisk_log_start":"424'6550","created":398,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:39:10.428693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26673152,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14295,"num_write_kb":470464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2140,"num_bytes_recovered":140247040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":1025,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014656+0000","last_change":"2024-09-14T04:24:07.974014+0000","last_active":"2024-09-14T04:28:43.014656+0000","last_peered":"2024-09-14T04:28:43.014656+0000","last_clean":"2024-09-14T04:28:43.014656+0000","last_became_active":"2024-09-14T04:24:07.973602+0000","last_became_peered":"2024-09-14T04:24:07.973602+0000","last_unstale":"2024-09-14T04:28:43.014656+0000","last_undegraded":"2024-09-14T04:28:43.014656+0000","last_fullsized":"2024-09-14T04:28:43.014656+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:36.915353+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":51,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010066+0000","last_change":"2024-09-14T04:27:20.248193+0000","last_active":"2024-09-14T04:28:44.010066+0000","last_peered":"2024-09-14T04:28:44.010066+0000","last_clean":"2024-09-14T04:28:44.010066+0000","last_became_active":"2024-09-14T04:27:20.247785+0000","last_became_peered":"2024-09-14T04:27:20.247785+0000","last_unstale":"2024-09-14T04:28:44.010066+0000","last_undegraded":"2024-09-14T04:28:44.010066+0000","last_fullsized":"2024-09-14T04:28:44.010066+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:44.759346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":811,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010168+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:28:44.010168+0000","last_peered":"2024-09-14T04:28:44.010168+0000","last_clean":"2024-09-14T04:28:44.010168+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:28:44.010168+0000","last_undegraded":"2024-09-14T04:28:44.010168+0000","last_fullsized":"2024-09-14T04:28:44.010168+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":573,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010810+0000","last_change":"2024-09-14T04:19:47.450022+0000","last_active":"2024-09-14T04:28:44.010810+0000","last_peered":"2024-09-14T04:28:44.010810+0000","last_clean":"2024-09-14T04:28:44.010810+0000","last_became_active":"2024-09-14T04:19:47.449185+0000","last_became_peered":"2024-09-14T04:19:47.449185+0000","last_unstale":"2024-09-14T04:28:44.010810+0000","last_undegraded":"2024-09-14T04:28:44.010810+0000","last_fullsized":"2024-09-14T04:28:44.010810+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:39.590448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"482'14163","reported_seq":16834,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.021499+0000","last_change":"2024-09-14T04:28:22.755027+0000","last_active":"2024-09-14T04:28:45.021499+0000","last_peered":"2024-09-14T04:28:45.021499+0000","last_clean":"2024-09-14T04:28:45.021499+0000","last_became_active":"2024-09-14T04:28:22.694782+0000","last_became_peered":"2024-09-14T04:28:22.694782+0000","last_unstale":"2024-09-14T04:28:45.021499+0000","last_undegraded":"2024-09-14T04:28:45.021499+0000","last_fullsized":"2024-09-14T04:28:45.021499+0000","mapping_epoch":463,"log_start":"427'7832","ondisk_log_start":"427'7832","created":398,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:24.712177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28246016,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14163,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1265,"num_bytes_recovered":82903040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":747,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010124+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:28:44.010124+0000","last_peered":"2024-09-14T04:28:44.010124+0000","last_clean":"2024-09-14T04:28:44.010124+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:28:44.010124+0000","last_undegraded":"2024-09-14T04:28:44.010124+0000","last_fullsized":"2024-09-14T04:28:44.010124+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":775,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.011806+0000","last_change":"2024-09-14T04:19:47.449436+0000","last_active":"2024-09-14T04:28:42.011806+0000","last_peered":"2024-09-14T04:28:42.011806+0000","last_clean":"2024-09-14T04:28:42.011806+0000","last_became_active":"2024-09-14T04:19:47.449134+0000","last_became_peered":"2024-09-14T04:19:47.449134+0000","last_unstale":"2024-09-14T04:28:42.011806+0000","last_undegraded":"2024-09-14T04:28:42.011806+0000","last_fullsized":"2024-09-14T04:28:42.011806+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:18.595994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":808,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014917+0000","last_change":"2024-09-14T04:19:47.440661+0000","last_active":"2024-09-14T04:28:43.014917+0000","last_peered":"2024-09-14T04:28:43.014917+0000","last_clean":"2024-09-14T04:28:43.014917+0000","last_became_active":"2024-09-14T04:19:47.440338+0000","last_became_peered":"2024-09-14T04:19:47.440338+0000","last_unstale":"2024-09-14T04:28:43.014917+0000","last_undegraded":"2024-09-14T04:28:43.014917+0000","last_fullsized":"2024-09-14T04:28:43.014917+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:14.218370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":276,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015059+0000","last_change":"2024-09-14T04:24:08.986781+0000","last_active":"2024-09-14T04:28:43.015059+0000","last_peered":"2024-09-14T04:28:43.015059+0000","last_clean":"2024-09-14T04:28:43.015059+0000","last_became_active":"2024-09-14T04:24:08.986322+0000","last_became_peered":"2024-09-14T04:24:08.986322+0000","last_unstale":"2024-09-14T04:28:43.015059+0000","last_undegraded":"2024-09-14T04:28:43.015059+0000","last_fullsized":"2024-09-14T04:28:43.015059+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:26.802101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"482'14524","reported_seq":19778,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.270330+0000","last_change":"2024-09-14T04:28:05.333945+0000","last_active":"2024-09-14T04:28:45.270330+0000","last_peered":"2024-09-14T04:28:45.270330+0000","last_clean":"2024-09-14T04:28:45.270330+0000","last_became_active":"2024-09-14T04:28:05.257907+0000","last_became_peered":"2024-09-14T04:28:05.257907+0000","last_unstale":"2024-09-14T04:28:45.270330+0000","last_undegraded":"2024-09-14T04:28:45.270330+0000","last_fullsized":"2024-09-14T04:28:45.270330+0000","mapping_epoch":456,"log_start":"428'9045","ondisk_log_start":"428'9045","created":398,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5479,"log_dups_size":0,"ondisk_log_size":5479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:37:37.719619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27328536,"num_objects":418,"num_object_clones":0,"num_object_copies":1254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":418,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14524,"num_write_kb":478081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2567,"num_bytes_recovered":168099864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":707,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999408+0000","last_change":"2024-09-14T04:24:47.566597+0000","last_active":"2024-09-14T04:28:39.999408+0000","last_peered":"2024-09-14T04:28:39.999408+0000","last_clean":"2024-09-14T04:28:39.999408+0000","last_became_active":"2024-09-14T04:24:07.995178+0000","last_became_peered":"2024-09-14T04:24:07.995178+0000","last_unstale":"2024-09-14T04:28:39.999408+0000","last_undegraded":"2024-09-14T04:28:39.999408+0000","last_fullsized":"2024-09-14T04:28:39.999408+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:47.566496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:47.566496+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:45.662588+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":415,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999120+0000","last_change":"2024-09-14T04:24:07.984719+0000","last_active":"2024-09-14T04:28:39.999120+0000","last_peered":"2024-09-14T04:28:39.999120+0000","last_clean":"2024-09-14T04:28:39.999120+0000","last_became_active":"2024-09-14T04:24:07.984367+0000","last_became_peered":"2024-09-14T04:24:07.984367+0000","last_unstale":"2024-09-14T04:28:39.999120+0000","last_undegraded":"2024-09-14T04:28:39.999120+0000","last_fullsized":"2024-09-14T04:28:39.999120+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:19.388893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":511,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014427+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:28:44.014427+0000","last_peered":"2024-09-14T04:28:44.014427+0000","last_clean":"2024-09-14T04:28:44.014427+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:28:44.014427+0000","last_undegraded":"2024-09-14T04:28:44.014427+0000","last_fullsized":"2024-09-14T04:28:44.014427+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.001374+0000","last_change":"2024-09-14T04:24:07.987411+0000","last_active":"2024-09-14T04:28:40.001374+0000","last_peered":"2024-09-14T04:28:40.001374+0000","last_clean":"2024-09-14T04:28:40.001374+0000","last_became_active":"2024-09-14T04:24:07.987104+0000","last_became_peered":"2024-09-14T04:24:07.987104+0000","last_unstale":"2024-09-14T04:28:40.001374+0000","last_undegraded":"2024-09-14T04:28:40.001374+0000","last_fullsized":"2024-09-14T04:28:40.001374+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:24:40.868920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.b","version":"477'14484","reported_seq":14574,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.633313+0000","last_change":"2024-09-14T04:25:02.146582+0000","last_active":"2024-09-14T04:28:40.633313+0000","last_peered":"2024-09-14T04:28:40.633313+0000","last_clean":"2024-09-14T04:28:40.633313+0000","last_became_active":"2024-09-14T04:25:02.146157+0000","last_became_peered":"2024-09-14T04:25:02.146157+0000","last_unstale":"2024-09-14T04:28:40.633313+0000","last_undegraded":"2024-09-14T04:28:40.633313+0000","last_fullsized":"2024-09-14T04:28:40.633313+0000","mapping_epoch":398,"log_start":"431'9184","ondisk_log_start":"431'9184","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:00.755296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32243712,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14484,"num_write_kb":479232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":803,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.005231+0000","last_change":"2024-09-14T04:27:23.264670+0000","last_active":"2024-09-14T04:28:41.005231+0000","last_peered":"2024-09-14T04:28:41.005231+0000","last_clean":"2024-09-14T04:28:41.005231+0000","last_became_active":"2024-09-14T04:27:23.264241+0000","last_became_peered":"2024-09-14T04:27:23.264241+0000","last_unstale":"2024-09-14T04:28:41.005231+0000","last_undegraded":"2024-09-14T04:28:41.005231+0000","last_fullsized":"2024-09-14T04:28:41.005231+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:27.406676+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":444,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.999965+0000","last_change":"2024-09-14T04:24:07.987026+0000","last_active":"2024-09-14T04:28:39.999965+0000","last_peered":"2024-09-14T04:28:39.999965+0000","last_clean":"2024-09-14T04:28:39.999965+0000","last_became_active":"2024-09-14T04:24:07.986717+0000","last_became_peered":"2024-09-14T04:24:07.986717+0000","last_unstale":"2024-09-14T04:28:39.999965+0000","last_undegraded":"2024-09-14T04:28:39.999965+0000","last_fullsized":"2024-09-14T04:28:39.999965+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:36.729024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":872,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015413+0000","last_change":"2024-09-14T04:27:20.252461+0000","last_active":"2024-09-14T04:28:43.015413+0000","last_peered":"2024-09-14T04:28:43.015413+0000","last_clean":"2024-09-14T04:28:43.015413+0000","last_became_active":"2024-09-14T04:27:20.252036+0000","last_became_peered":"2024-09-14T04:27:20.252036+0000","last_unstale":"2024-09-14T04:28:43.015413+0000","last_undegraded":"2024-09-14T04:28:43.015413+0000","last_fullsized":"2024-09-14T04:28:43.015413+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:32:51.681202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":291,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014749+0000","last_change":"2024-09-14T04:25:25.578605+0000","last_active":"2024-09-14T04:28:43.014749+0000","last_peered":"2024-09-14T04:28:43.014749+0000","last_clean":"2024-09-14T04:28:43.014749+0000","last_became_active":"2024-09-14T04:25:25.578270+0000","last_became_peered":"2024-09-14T04:25:25.578270+0000","last_unstale":"2024-09-14T04:28:43.014749+0000","last_undegraded":"2024-09-14T04:28:43.014749+0000","last_fullsized":"2024-09-14T04:28:43.014749+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:42.335968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"479'14121","reported_seq":14307,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.569015+0000","last_change":"2024-09-14T04:25:01.995325+0000","last_active":"2024-09-14T04:28:42.569015+0000","last_peered":"2024-09-14T04:28:42.569015+0000","last_clean":"2024-09-14T04:28:42.569015+0000","last_became_active":"2024-09-14T04:25:01.995051+0000","last_became_peered":"2024-09-14T04:25:01.995051+0000","last_unstale":"2024-09-14T04:28:42.569015+0000","last_undegraded":"2024-09-14T04:28:42.569015+0000","last_fullsized":"2024-09-14T04:28:42.569015+0000","mapping_epoch":398,"log_start":"426'6955","ondisk_log_start":"426'6955","created":398,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:25:00.974891+0000","objects_scrubbed":0,"log_size":7166,"log_dups_size":0,"ondisk_log_size":7166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:19.886703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32178176,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14121,"num_write_kb":467584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":805,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010147+0000","last_change":"2024-09-14T04:28:22.699361+0000","last_active":"2024-09-14T04:28:44.010147+0000","last_peered":"2024-09-14T04:28:44.010147+0000","last_clean":"2024-09-14T04:28:44.010147+0000","last_became_active":"2024-09-14T04:28:22.699020+0000","last_became_peered":"2024-09-14T04:28:22.699020+0000","last_unstale":"2024-09-14T04:28:44.010147+0000","last_undegraded":"2024-09-14T04:28:44.010147+0000","last_fullsized":"2024-09-14T04:28:44.010147+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:07.313428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":811,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015393+0000","last_change":"2024-09-14T04:24:07.987737+0000","last_active":"2024-09-14T04:28:43.015393+0000","last_peered":"2024-09-14T04:28:43.015393+0000","last_clean":"2024-09-14T04:28:43.015393+0000","last_became_active":"2024-09-14T04:24:07.987422+0000","last_became_peered":"2024-09-14T04:24:07.987422+0000","last_unstale":"2024-09-14T04:28:43.015393+0000","last_undegraded":"2024-09-14T04:28:43.015393+0000","last_fullsized":"2024-09-14T04:28:43.015393+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:06.094462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":51,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010962+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:28:44.010962+0000","last_peered":"2024-09-14T04:28:44.010962+0000","last_clean":"2024-09-14T04:28:44.010962+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:28:44.010962+0000","last_undegraded":"2024-09-14T04:28:44.010962+0000","last_fullsized":"2024-09-14T04:28:44.010962+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":585,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010790+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:28:44.010790+0000","last_peered":"2024-09-14T04:28:44.010790+0000","last_clean":"2024-09-14T04:28:44.010790+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:28:44.010790+0000","last_undegraded":"2024-09-14T04:28:44.010790+0000","last_fullsized":"2024-09-14T04:28:44.010790+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"477'14094","reported_seq":14410,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.656149+0000","last_change":"2024-09-14T04:27:16.935967+0000","last_active":"2024-09-14T04:28:40.656149+0000","last_peered":"2024-09-14T04:28:40.656149+0000","last_clean":"2024-09-14T04:28:40.656149+0000","last_became_active":"2024-09-14T04:27:04.683999+0000","last_became_peered":"2024-09-14T04:27:04.683999+0000","last_unstale":"2024-09-14T04:28:40.656149+0000","last_undegraded":"2024-09-14T04:28:40.656149+0000","last_fullsized":"2024-09-14T04:28:40.656149+0000","mapping_epoch":398,"log_start":"428'8733","ondisk_log_start":"428'8733","created":398,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"438'11855","last_scrub_stamp":"2024-09-14T04:27:16.935785+0000","last_deep_scrub":"438'11855","last_deep_scrub_stamp":"2024-09-14T04:27:16.935785+0000","last_clean_scrub_stamp":"2024-09-14T04:27:16.935785+0000","objects_scrubbed":2574,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:33.473629+0000","scrub_duration":7.8014888280000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17956864,"num_objects":274,"num_object_clones":0,"num_object_copies":822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2300,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":788,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014499+0000","last_change":"2024-09-14T04:24:08.982544+0000","last_active":"2024-09-14T04:28:44.014499+0000","last_peered":"2024-09-14T04:28:44.014499+0000","last_clean":"2024-09-14T04:28:44.014499+0000","last_became_active":"2024-09-14T04:24:08.982132+0000","last_became_peered":"2024-09-14T04:24:08.982132+0000","last_unstale":"2024-09-14T04:28:44.014499+0000","last_undegraded":"2024-09-14T04:28:44.014499+0000","last_fullsized":"2024-09-14T04:28:44.014499+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:30.440042+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":55,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011383+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:28:44.011383+0000","last_peered":"2024-09-14T04:28:44.011383+0000","last_clean":"2024-09-14T04:28:44.011383+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:28:44.011383+0000","last_undegraded":"2024-09-14T04:28:44.011383+0000","last_fullsized":"2024-09-14T04:28:44.011383+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":804,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.015805+0000","last_change":"2024-09-14T04:28:19.257534+0000","last_active":"2024-09-14T04:28:43.015805+0000","last_peered":"2024-09-14T04:28:43.015805+0000","last_clean":"2024-09-14T04:28:43.015805+0000","last_became_active":"2024-09-14T04:28:19.257051+0000","last_became_peered":"2024-09-14T04:28:19.257051+0000","last_unstale":"2024-09-14T04:28:43.015805+0000","last_undegraded":"2024-09-14T04:28:43.015805+0000","last_fullsized":"2024-09-14T04:28:43.015805+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:42.727253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":51,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010331+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:28:44.010331+0000","last_peered":"2024-09-14T04:28:44.010331+0000","last_clean":"2024-09-14T04:28:44.010331+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:28:44.010331+0000","last_undegraded":"2024-09-14T04:28:44.010331+0000","last_fullsized":"2024-09-14T04:28:44.010331+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"479'11511","reported_seq":12477,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.550397+0000","last_change":"2024-09-14T04:28:33.707722+0000","last_active":"2024-09-14T04:28:42.550397+0000","last_peered":"2024-09-14T04:28:42.550397+0000","last_clean":"2024-09-14T04:28:42.550397+0000","last_became_active":"2024-09-14T04:28:33.689698+0000","last_became_peered":"2024-09-14T04:28:33.689698+0000","last_unstale":"2024-09-14T04:28:42.550397+0000","last_undegraded":"2024-09-14T04:28:42.550397+0000","last_fullsized":"2024-09-14T04:28:42.550397+0000","mapping_epoch":470,"log_start":"410'2400","ondisk_log_start":"410'2400","created":398,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"431'9002","last_scrub_stamp":"2024-09-14T04:27:06.233137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:06.233137+0000","objects_scrubbed":2784,"log_size":9111,"log_dups_size":0,"ondisk_log_size":9111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:22.249722+0000","scrub_duration":1174,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121856,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2538,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":360,"num_bytes_recovered":23592960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1092,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.005129+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:28:41.005129+0000","last_peered":"2024-09-14T04:28:41.005129+0000","last_clean":"2024-09-14T04:28:41.005129+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:28:41.005129+0000","last_undegraded":"2024-09-14T04:28:41.005129+0000","last_fullsized":"2024-09-14T04:28:41.005129+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":577,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010594+0000","last_change":"2024-09-14T04:25:52.604247+0000","last_active":"2024-09-14T04:28:44.010594+0000","last_peered":"2024-09-14T04:28:44.010594+0000","last_clean":"2024-09-14T04:28:44.010594+0000","last_became_active":"2024-09-14T04:25:52.591054+0000","last_became_peered":"2024-09-14T04:25:52.591054+0000","last_unstale":"2024-09-14T04:28:44.010594+0000","last_undegraded":"2024-09-14T04:28:44.010594+0000","last_fullsized":"2024-09-14T04:28:44.010594+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:15.295970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":32,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010870+0000","last_change":"2024-09-14T04:28:19.279877+0000","last_active":"2024-09-14T04:28:44.010870+0000","last_peered":"2024-09-14T04:28:44.010870+0000","last_clean":"2024-09-14T04:28:44.010870+0000","last_became_active":"2024-09-14T04:28:19.279587+0000","last_became_peered":"2024-09-14T04:28:19.279587+0000","last_unstale":"2024-09-14T04:28:44.010870+0000","last_undegraded":"2024-09-14T04:28:44.010870+0000","last_fullsized":"2024-09-14T04:28:44.010870+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:39.370051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":477,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014871+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:28:44.014871+0000","last_peered":"2024-09-14T04:28:44.014871+0000","last_clean":"2024-09-14T04:28:44.014871+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:28:44.014871+0000","last_undegraded":"2024-09-14T04:28:44.014871+0000","last_fullsized":"2024-09-14T04:28:44.014871+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":513,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014195+0000","last_change":"2024-09-14T04:28:19.257327+0000","last_active":"2024-09-14T04:28:44.014195+0000","last_peered":"2024-09-14T04:28:44.014195+0000","last_clean":"2024-09-14T04:28:44.014195+0000","last_became_active":"2024-09-14T04:28:19.256940+0000","last_became_peered":"2024-09-14T04:28:19.256940+0000","last_unstale":"2024-09-14T04:28:44.014195+0000","last_undegraded":"2024-09-14T04:28:44.014195+0000","last_fullsized":"2024-09-14T04:28:44.014195+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:07:10.308714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":541,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.014034+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:28:44.014034+0000","last_peered":"2024-09-14T04:28:44.014034+0000","last_clean":"2024-09-14T04:28:44.014034+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:28:44.014034+0000","last_undegraded":"2024-09-14T04:28:44.014034+0000","last_fullsized":"2024-09-14T04:28:44.014034+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":262,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014636+0000","last_change":"2024-09-14T04:23:35.659759+0000","last_active":"2024-09-14T04:28:43.014636+0000","last_peered":"2024-09-14T04:28:43.014636+0000","last_clean":"2024-09-14T04:28:43.014636+0000","last_became_active":"2024-09-14T04:23:35.659449+0000","last_became_peered":"2024-09-14T04:23:35.659449+0000","last_unstale":"2024-09-14T04:28:43.014636+0000","last_undegraded":"2024-09-14T04:28:43.014636+0000","last_fullsized":"2024-09-14T04:28:43.014636+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:41.733303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":57,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010871+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:28:44.010871+0000","last_peered":"2024-09-14T04:28:44.010871+0000","last_clean":"2024-09-14T04:28:44.010871+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:28:44.010871+0000","last_undegraded":"2024-09-14T04:28:44.010871+0000","last_fullsized":"2024-09-14T04:28:44.010871+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.18","version":"481'11508","reported_seq":13290,"reported_epoch":482,"state":"peering","last_fresh":"2024-09-14T04:28:45.018944+0000","last_change":"2024-09-14T04:28:45.018944+0000","last_active":"2024-09-14T04:28:45.018944+0000","last_peered":"2024-09-14T04:28:44.828162+0000","last_clean":"2024-09-14T04:28:41.008732+0000","last_became_active":"2024-09-14T04:28:44.114310+0000","last_became_peered":"2024-09-14T04:28:44.114310+0000","last_unstale":"2024-09-14T04:28:45.018944+0000","last_undegraded":"2024-09-14T04:28:45.018944+0000","last_fullsized":"2024-09-14T04:28:45.018944+0000","mapping_epoch":482,"log_start":"410'2400","ondisk_log_start":"410'2400","created":430,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9141","last_scrub_stamp":"2024-09-14T04:27:09.911112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:09.911112+0000","objects_scrubbed":2594,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:09.863977+0000","scrub_duration":784,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2380,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":548,"num_bytes_recovered":35913728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":1029,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.012393+0000","last_change":"2024-09-14T04:27:23.254868+0000","last_active":"2024-09-14T04:28:42.012393+0000","last_peered":"2024-09-14T04:28:42.012393+0000","last_clean":"2024-09-14T04:28:42.012393+0000","last_became_active":"2024-09-14T04:27:23.254269+0000","last_became_peered":"2024-09-14T04:27:23.254269+0000","last_unstale":"2024-09-14T04:28:42.012393+0000","last_undegraded":"2024-09-14T04:28:42.012393+0000","last_fullsized":"2024-09-14T04:28:42.012393+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:06:43.559813+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":379,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.000798+0000","last_change":"2024-09-14T04:27:20.244084+0000","last_active":"2024-09-14T04:28:40.000798+0000","last_peered":"2024-09-14T04:28:40.000798+0000","last_clean":"2024-09-14T04:28:40.000798+0000","last_became_active":"2024-09-14T04:27:20.243703+0000","last_became_peered":"2024-09-14T04:27:20.243703+0000","last_unstale":"2024-09-14T04:28:40.000798+0000","last_undegraded":"2024-09-14T04:28:40.000798+0000","last_fullsized":"2024-09-14T04:28:40.000798+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:34.342306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010585+0000","last_change":"2024-09-14T04:28:19.257505+0000","last_active":"2024-09-14T04:28:44.010585+0000","last_peered":"2024-09-14T04:28:44.010585+0000","last_clean":"2024-09-14T04:28:44.010585+0000","last_became_active":"2024-09-14T04:28:19.257193+0000","last_became_peered":"2024-09-14T04:28:19.257193+0000","last_unstale":"2024-09-14T04:28:44.010585+0000","last_undegraded":"2024-09-14T04:28:44.010585+0000","last_fullsized":"2024-09-14T04:28:44.010585+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:03:33.146736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.19","version":"477'11525","reported_seq":11684,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.673467+0000","last_change":"2024-09-14T04:27:22.628893+0000","last_active":"2024-09-14T04:28:40.673467+0000","last_peered":"2024-09-14T04:28:40.673467+0000","last_clean":"2024-09-14T04:28:40.673467+0000","last_became_active":"2024-09-14T04:27:04.697978+0000","last_became_peered":"2024-09-14T04:27:04.697978+0000","last_unstale":"2024-09-14T04:28:40.673467+0000","last_undegraded":"2024-09-14T04:28:40.673467+0000","last_fullsized":"2024-09-14T04:28:40.673467+0000","mapping_epoch":430,"log_start":"420'3618","ondisk_log_start":"420'3618","created":430,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":5,"last_scrub":"444'9315","last_scrub_stamp":"2024-09-14T04:27:22.628541+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:22.628541+0000","objects_scrubbed":2476,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:05.021923+0000","scrub_duration":0.47792231099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16711680,"num_objects":255,"num_object_clones":0,"num_object_copies":765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2221,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":572,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011345+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:28:44.011345+0000","last_peered":"2024-09-14T04:28:44.011345+0000","last_clean":"2024-09-14T04:28:44.011345+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:28:44.011345+0000","last_undegraded":"2024-09-14T04:28:44.011345+0000","last_fullsized":"2024-09-14T04:28:44.011345+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":670,"reported_epoch":478,"state":"active+clean","last_fresh":"2024-09-14T04:28:41.004788+0000","last_change":"2024-09-14T04:14:43.298708+0000","last_active":"2024-09-14T04:28:41.004788+0000","last_peered":"2024-09-14T04:28:41.004788+0000","last_clean":"2024-09-14T04:28:41.004788+0000","last_became_active":"2024-09-14T04:14:43.298205+0000","last_became_peered":"2024-09-14T04:14:43.298205+0000","last_unstale":"2024-09-14T04:28:41.004788+0000","last_undegraded":"2024-09-14T04:28:41.004788+0000","last_fullsized":"2024-09-14T04:28:41.004788+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:28.924384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.16","version":"477'11670","reported_seq":12318,"reported_epoch":478,"state":"peering","last_fresh":"2024-09-14T04:28:41.005845+0000","last_change":"2024-09-14T04:28:41.005845+0000","last_active":"2024-09-14T04:28:41.005845+0000","last_peered":"2024-09-14T04:28:40.664204+0000","last_clean":"2024-09-14T04:28:40.664204+0000","last_became_active":"2024-09-14T04:28:40.076625+0000","last_became_peered":"2024-09-14T04:28:40.076625+0000","last_unstale":"2024-09-14T04:28:41.005845+0000","last_undegraded":"2024-09-14T04:28:41.005845+0000","last_fullsized":"2024-09-14T04:28:41.005845+0000","mapping_epoch":478,"log_start":"420'3900","ondisk_log_start":"420'3900","created":430,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":5,"last_scrub":"445'9522","last_scrub_stamp":"2024-09-14T04:27:24.500816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:24.500816+0000","objects_scrubbed":2391,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.56814457399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2155,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":238,"num_bytes_recovered":15597568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":32,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011042+0000","last_change":"2024-09-14T04:28:19.260335+0000","last_active":"2024-09-14T04:28:44.011042+0000","last_peered":"2024-09-14T04:28:44.011042+0000","last_clean":"2024-09-14T04:28:44.011042+0000","last_became_active":"2024-09-14T04:28:19.260004+0000","last_became_peered":"2024-09-14T04:28:19.260004+0000","last_unstale":"2024-09-14T04:28:44.011042+0000","last_undegraded":"2024-09-14T04:28:44.011042+0000","last_fullsized":"2024-09-14T04:28:44.011042+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:53.588795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":68,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:39.998995+0000","last_change":"2024-09-14T04:25:52.582350+0000","last_active":"2024-09-14T04:28:39.998995+0000","last_peered":"2024-09-14T04:28:39.998995+0000","last_clean":"2024-09-14T04:28:39.998995+0000","last_became_active":"2024-09-14T04:25:52.582042+0000","last_became_peered":"2024-09-14T04:25:52.582042+0000","last_unstale":"2024-09-14T04:28:39.998995+0000","last_undegraded":"2024-09-14T04:28:39.998995+0000","last_fullsized":"2024-09-14T04:28:39.998995+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:00.528145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.17","version":"472'11671","reported_seq":18765,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-09-14T04:28:34.783884+0000","last_change":"2024-09-14T04:28:30.698418+0000","last_active":"2024-09-14T04:28:34.783884+0000","last_peered":"2024-09-14T04:28:34.783884+0000","last_clean":"2024-09-14T04:28:34.783884+0000","last_became_active":"2024-09-14T04:28:30.668399+0000","last_became_peered":"2024-09-14T04:28:30.668399+0000","last_unstale":"2024-09-14T04:28:34.783884+0000","last_undegraded":"2024-09-14T04:28:34.783884+0000","last_fullsized":"2024-09-14T04:28:34.783884+0000","mapping_epoch":467,"log_start":"420'4100","ondisk_log_start":"420'4100","created":430,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":5,"last_scrub":"433'9489","last_scrub_stamp":"2024-09-14T04:27:12.513993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:12.513993+0000","objects_scrubbed":2501,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:56.991088+0000","scrub_duration":0.43811768600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20578304,"num_objects":314,"num_object_clones":0,"num_object_copies":942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2187,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1089,"num_bytes_recovered":71303168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":596,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010486+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:28:44.010486+0000","last_peered":"2024-09-14T04:28:44.010486+0000","last_clean":"2024-09-14T04:28:44.010486+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:28:44.010486+0000","last_undegraded":"2024-09-14T04:28:44.010486+0000","last_fullsized":"2024-09-14T04:28:44.010486+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":834,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.014516+0000","last_change":"2024-09-14T04:25:07.959001+0000","last_active":"2024-09-14T04:28:43.014516+0000","last_peered":"2024-09-14T04:28:43.014516+0000","last_clean":"2024-09-14T04:28:43.014516+0000","last_became_active":"2024-09-14T04:25:07.958592+0000","last_became_peered":"2024-09-14T04:25:07.958592+0000","last_unstale":"2024-09-14T04:28:43.014516+0000","last_undegraded":"2024-09-14T04:28:43.014516+0000","last_fullsized":"2024-09-14T04:28:43.014516+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:57.488370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.14","version":"481'11851","reported_seq":13800,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.592290+0000","last_change":"2024-09-14T04:28:31.705237+0000","last_active":"2024-09-14T04:28:44.592290+0000","last_peered":"2024-09-14T04:28:44.592290+0000","last_clean":"2024-09-14T04:28:44.592290+0000","last_became_active":"2024-09-14T04:28:31.677855+0000","last_became_peered":"2024-09-14T04:28:31.677855+0000","last_unstale":"2024-09-14T04:28:44.592290+0000","last_undegraded":"2024-09-14T04:28:44.592290+0000","last_fullsized":"2024-09-14T04:28:44.592290+0000","mapping_epoch":468,"log_start":"412'2700","ondisk_log_start":"412'2700","created":430,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9492","last_scrub_stamp":"2024-09-14T04:27:11.031534+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:11.031534+0000","objects_scrubbed":2607,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:39.271441+0000","scrub_duration":951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2371,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":424,"num_bytes_recovered":27787264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":328,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.000551+0000","last_change":"2024-09-14T04:24:07.978512+0000","last_active":"2024-09-14T04:28:40.000551+0000","last_peered":"2024-09-14T04:28:40.000551+0000","last_clean":"2024-09-14T04:28:40.000551+0000","last_became_active":"2024-09-14T04:24:07.978174+0000","last_became_peered":"2024-09-14T04:24:07.978174+0000","last_unstale":"2024-09-14T04:28:40.000551+0000","last_undegraded":"2024-09-14T04:28:40.000551+0000","last_fullsized":"2024-09-14T04:28:40.000551+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:18:38.328494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":72,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.015048+0000","last_change":"2024-09-14T04:25:52.594984+0000","last_active":"2024-09-14T04:28:44.015048+0000","last_peered":"2024-09-14T04:28:44.015048+0000","last_clean":"2024-09-14T04:28:44.015048+0000","last_became_active":"2024-09-14T04:25:52.594627+0000","last_became_peered":"2024-09-14T04:25:52.594627+0000","last_unstale":"2024-09-14T04:28:44.015048+0000","last_undegraded":"2024-09-14T04:28:44.015048+0000","last_fullsized":"2024-09-14T04:28:44.015048+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:35:51.582615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.15","version":"482'11869","reported_seq":12557,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.200592+0000","last_change":"2024-09-14T04:28:40.026420+0000","last_active":"2024-09-14T04:28:45.200592+0000","last_peered":"2024-09-14T04:28:45.200592+0000","last_clean":"2024-09-14T04:28:45.200592+0000","last_became_active":"2024-09-14T04:28:40.006136+0000","last_became_peered":"2024-09-14T04:28:40.006136+0000","last_unstale":"2024-09-14T04:28:45.200592+0000","last_undegraded":"2024-09-14T04:28:45.200592+0000","last_fullsized":"2024-09-14T04:28:45.200592+0000","mapping_epoch":476,"log_start":"420'4100","ondisk_log_start":"420'4100","created":430,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'9476","last_scrub_stamp":"2024-09-14T04:27:11.636212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:11.636212+0000","objects_scrubbed":2601,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:28.504300+0000","scrub_duration":0.52699493200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13238272,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2399,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":249,"num_bytes_recovered":16318464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":243,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.012507+0000","last_change":"2024-09-14T04:23:47.747851+0000","last_active":"2024-09-14T04:28:42.012507+0000","last_peered":"2024-09-14T04:28:42.012507+0000","last_clean":"2024-09-14T04:28:42.012507+0000","last_became_active":"2024-09-14T04:23:47.747515+0000","last_became_peered":"2024-09-14T04:23:47.747515+0000","last_unstale":"2024-09-14T04:28:42.012507+0000","last_undegraded":"2024-09-14T04:28:42.012507+0000","last_fullsized":"2024-09-14T04:28:42.012507+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:28.662906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":593,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.010687+0000","last_change":"2024-09-14T04:27:20.298495+0000","last_active":"2024-09-14T04:28:44.010687+0000","last_peered":"2024-09-14T04:28:44.010687+0000","last_clean":"2024-09-14T04:28:44.010687+0000","last_became_active":"2024-09-14T04:27:20.297682+0000","last_became_peered":"2024-09-14T04:27:20.297682+0000","last_unstale":"2024-09-14T04:28:44.010687+0000","last_undegraded":"2024-09-14T04:28:44.010687+0000","last_fullsized":"2024-09-14T04:28:44.010687+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:02.663766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.12","version":"481'11722","reported_seq":24856,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.500485+0000","last_change":"2024-09-14T04:27:40.246292+0000","last_active":"2024-09-14T04:28:44.500485+0000","last_peered":"2024-09-14T04:28:44.500485+0000","last_clean":"2024-09-14T04:28:44.500485+0000","last_became_active":"2024-09-14T04:27:40.234528+0000","last_became_peered":"2024-09-14T04:27:40.234528+0000","last_unstale":"2024-09-14T04:28:44.500485+0000","last_undegraded":"2024-09-14T04:28:44.500485+0000","last_fullsized":"2024-09-14T04:28:44.500485+0000","mapping_epoch":448,"log_start":"416'3200","ondisk_log_start":"416'3200","created":430,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":5,"last_scrub":"433'9342","last_scrub_stamp":"2024-09-14T04:27:12.514256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:12.514256+0000","objects_scrubbed":2588,"log_size":8522,"log_dups_size":0,"ondisk_log_size":8522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:23.374804+0000","scrub_duration":0.39924970199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13500416,"num_objects":206,"num_object_clones":0,"num_object_copies":618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2382,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2257,"num_bytes_recovered":147914752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.013901+0000","last_change":"2024-09-14T04:25:07.963986+0000","last_active":"2024-09-14T04:28:44.013901+0000","last_peered":"2024-09-14T04:28:44.013901+0000","last_clean":"2024-09-14T04:28:44.013901+0000","last_became_active":"2024-09-14T04:25:07.963503+0000","last_became_peered":"2024-09-14T04:25:07.963503+0000","last_unstale":"2024-09-14T04:28:44.013901+0000","last_undegraded":"2024-09-14T04:28:44.013901+0000","last_fullsized":"2024-09-14T04:28:44.013901+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:37.006919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":426,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.001214+0000","last_change":"2024-09-14T04:24:07.974941+0000","last_active":"2024-09-14T04:28:40.001214+0000","last_peered":"2024-09-14T04:28:40.001214+0000","last_clean":"2024-09-14T04:28:40.001214+0000","last_became_active":"2024-09-14T04:24:07.974561+0000","last_became_peered":"2024-09-14T04:24:07.974561+0000","last_unstale":"2024-09-14T04:28:40.001214+0000","last_undegraded":"2024-09-14T04:28:40.001214+0000","last_fullsized":"2024-09-14T04:28:40.001214+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:58:55.892494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.13","version":"482'11774","reported_seq":18791,"reported_epoch":482,"state":"active+clean","last_fresh":"2024-09-14T04:28:45.018519+0000","last_change":"2024-09-14T04:27:39.256095+0000","last_active":"2024-09-14T04:28:45.018519+0000","last_peered":"2024-09-14T04:28:45.018519+0000","last_clean":"2024-09-14T04:28:45.018519+0000","last_became_active":"2024-09-14T04:27:39.235009+0000","last_became_peered":"2024-09-14T04:27:39.235009+0000","last_unstale":"2024-09-14T04:28:45.018519+0000","last_undegraded":"2024-09-14T04:28:45.018519+0000","last_fullsized":"2024-09-14T04:28:45.018519+0000","mapping_epoch":447,"log_start":"418'3600","ondisk_log_start":"418'3600","created":430,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"431'9232","last_scrub_stamp":"2024-09-14T04:27:07.385967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.385967+0000","objects_scrubbed":2775,"log_size":8174,"log_dups_size":0,"ondisk_log_size":8174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:46.090117+0000","scrub_duration":798,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14352384,"num_objects":219,"num_object_clones":0,"num_object_copies":657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2556,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2270,"num_bytes_recovered":148766720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":92,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.015290+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:28:44.015290+0000","last_peered":"2024-09-14T04:28:44.015290+0000","last_clean":"2024-09-14T04:28:44.015290+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:28:44.015290+0000","last_undegraded":"2024-09-14T04:28:44.015290+0000","last_fullsized":"2024-09-14T04:28:44.015290+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":435,"reported_epoch":477,"state":"active+clean","last_fresh":"2024-09-14T04:28:40.000429+0000","last_change":"2024-09-14T04:25:52.606053+0000","last_active":"2024-09-14T04:28:40.000429+0000","last_peered":"2024-09-14T04:28:40.000429+0000","last_clean":"2024-09-14T04:28:40.000429+0000","last_became_active":"2024-09-14T04:25:52.584747+0000","last_became_peered":"2024-09-14T04:25:52.584747+0000","last_unstale":"2024-09-14T04:28:40.000429+0000","last_undegraded":"2024-09-14T04:28:40.000429+0000","last_fullsized":"2024-09-14T04:28:40.000429+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:04.702137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.10","version":"480'11771","reported_seq":17122,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.452011+0000","last_change":"2024-09-14T04:27:20.260205+0000","last_active":"2024-09-14T04:28:43.452011+0000","last_peered":"2024-09-14T04:28:43.452011+0000","last_clean":"2024-09-14T04:28:43.452011+0000","last_became_active":"2024-09-14T04:27:20.242343+0000","last_became_peered":"2024-09-14T04:27:20.242343+0000","last_unstale":"2024-09-14T04:28:43.452011+0000","last_undegraded":"2024-09-14T04:28:43.452011+0000","last_fullsized":"2024-09-14T04:28:43.452011+0000","mapping_epoch":441,"log_start":"420'3900","ondisk_log_start":"420'3900","created":430,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"431'9216","last_scrub_stamp":"2024-09-14T04:27:07.003313+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:07.003313+0000","objects_scrubbed":2796,"log_size":7871,"log_dups_size":0,"ondisk_log_size":7871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:43.902091+0000","scrub_duration":928,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15007744,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2567,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2554,"num_bytes_recovered":167378944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":721,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.011768+0000","last_change":"2024-09-14T04:19:47.449334+0000","last_active":"2024-09-14T04:28:42.011768+0000","last_peered":"2024-09-14T04:28:42.011768+0000","last_clean":"2024-09-14T04:28:42.011768+0000","last_became_active":"2024-09-14T04:19:47.449027+0000","last_became_peered":"2024-09-14T04:19:47.449027+0000","last_unstale":"2024-09-14T04:28:42.011768+0000","last_undegraded":"2024-09-14T04:28:42.011768+0000","last_fullsized":"2024-09-14T04:28:42.011768+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:52.391407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":393,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-14T04:28:42.011421+0000","last_change":"2024-09-14T04:19:47.447318+0000","last_active":"2024-09-14T04:28:42.011421+0000","last_peered":"2024-09-14T04:28:42.011421+0000","last_clean":"2024-09-14T04:28:42.011421+0000","last_became_active":"2024-09-14T04:19:47.446432+0000","last_became_peered":"2024-09-14T04:19:47.446432+0000","last_unstale":"2024-09-14T04:28:42.011421+0000","last_undegraded":"2024-09-14T04:28:42.011421+0000","last_fullsized":"2024-09-14T04:28:42.011421+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:27:24.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"480'11350","reported_seq":17363,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-14T04:28:43.352141+0000","last_change":"2024-09-14T04:28:34.727976+0000","last_active":"2024-09-14T04:28:43.352141+0000","last_peered":"2024-09-14T04:28:43.352141+0000","last_clean":"2024-09-14T04:28:43.352141+0000","last_became_active":"2024-09-14T04:28:34.690234+0000","last_became_peered":"2024-09-14T04:28:34.690234+0000","last_unstale":"2024-09-14T04:28:43.352141+0000","last_undegraded":"2024-09-14T04:28:43.352141+0000","last_fullsized":"2024-09-14T04:28:43.352141+0000","mapping_epoch":471,"log_start":"409'2200","ondisk_log_start":"409'2200","created":430,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":5,"last_scrub":"432'8987","last_scrub_stamp":"2024-09-14T04:27:08.864358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:25:00.974891+0000","last_clean_scrub_stamp":"2024-09-14T04:27:08.864358+0000","objects_scrubbed":2611,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:24.409515+0000","scrub_duration":787,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14942208,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2383,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2039,"num_bytes_recovered":133627904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":465,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.013682+0000","last_change":"2024-09-14T04:14:43.300129+0000","last_active":"2024-09-14T04:28:44.013682+0000","last_peered":"2024-09-14T04:28:44.013682+0000","last_clean":"2024-09-14T04:28:44.013682+0000","last_became_active":"2024-09-14T04:14:43.299697+0000","last_became_peered":"2024-09-14T04:14:43.299697+0000","last_unstale":"2024-09-14T04:28:44.013682+0000","last_undegraded":"2024-09-14T04:28:44.013682+0000","last_fullsized":"2024-09-14T04:28:44.013682+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:27.073016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":620,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-14T04:28:44.011443+0000","last_change":"2024-09-14T04:24:09.006002+0000","last_active":"2024-09-14T04:28:44.011443+0000","last_peered":"2024-09-14T04:28:44.011443+0000","last_clean":"2024-09-14T04:28:44.011443+0000","last_became_active":"2024-09-14T04:24:09.005575+0000","last_became_peered":"2024-09-14T04:24:09.005575+0000","last_unstale":"2024-09-14T04:28:44.011443+0000","last_undegraded":"2024-09-14T04:28:44.011443+0000","last_fullsized":"2024-09-14T04:28:44.011443+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:45.184648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":486146072,"num_objects":7419,"num_object_clones":0,"num_object_copies":22257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7419,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134908,"num_write_kb":2843905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22720,"num_bytes_recovered":1488781336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1713258496,"data_stored":1713242208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200886,"ondisk_log_size":200886,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":21890,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2754144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201858,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1542988,"kb_used_data":183488,"kb_used_omap":1,"kb_used_meta":1359486,"kb_avail":92202164,"statfs":{"total":95995035648,"available":94415015936,"internally_reserved":0,"allocated":187891712,"data_stored":184737128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1392114123},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463496,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060792,"kb_used_data":123176,"kb_used_omap":1,"kb_used_meta":937598,"kb_avail":92684360,"statfs":{"total":95995035648,"available":94908784640,"internally_reserved":0,"allocated":126132224,"data_stored":123001492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":960100807},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834168,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719672,"kb_used_data":363308,"kb_used_omap":5,"kb_used_meta":356346,"kb_avail":93025480,"statfs":{"total":95995035648,"available":95258091520,"internally_reserved":0,"allocated":372027392,"data_stored":368920473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":364898365},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986768,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681736,"kb_used_data":224952,"kb_used_omap":1,"kb_used_meta":1456766,"kb_avail":92063416,"statfs":{"total":95995035648,"available":94272937984,"internally_reserved":0,"allocated":230350848,"data_stored":227204113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1491728838},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748796,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1256844,"kb_used_data":216892,"kb_used_omap":4,"kb_used_meta":1039931,"kb_avail":92488308,"statfs":{"total":95995035648,"available":94708027392,"internally_reserved":0,"allocated":222097408,"data_stored":218982349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1064889900},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605287017,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904988,"kb_used_data":293964,"kb_used_omap":10,"kb_used_meta":610869,"kb_avail":92840164,"statfs":{"total":95995035648,"available":95068327936,"internally_reserved":0,"allocated":301019136,"data_stored":297896989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":625530455},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468340,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292624,"kb_used_data":200196,"kb_used_omap":5,"kb_used_meta":1092410,"kb_avail":92452528,"statfs":{"total":95995035648,"available":94671388672,"internally_reserved":0,"allocated":205000704,"data_stored":201884212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1118628191},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342455,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924788,"kb_used_data":195180,"kb_used_omap":10,"kb_used_meta":729589,"kb_avail":92820364,"statfs":{"total":95995035648,"available":95048052736,"internally_reserved":0,"allocated":199864320,"data_stored":196725855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":747099346},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":194117632,"data_stored":194117632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":191762432,"data_stored":191758360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":284950528,"data_stored":284950528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":210571264,"data_stored":210567192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":208211968,"data_stored":208207896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":353501184,"data_stored":353501184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":98959360,"data_stored":98959360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":171184128,"data_stored":171180056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:28:50.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_8 pgp_num 26 2024-09-14T04:28:50.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.197 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:50.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:50.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:50.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:50.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[112165]: pgmap v1541: 119 pgs: 1 peering, 118 active+clean; 368 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:28:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:50.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[107399]: pgmap v1541: 119 pgs: 1 peering, 118 active+clean; 368 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:28:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:50.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:50.490 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:50.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:50 smithi012 ceph-mon[101180]: pgmap v1541: 119 pgs: 1 peering, 118 active+clean; 368 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-14T04:28:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:28:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:50.746 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:50.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:50.887 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:50.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:50.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:51.043 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:51.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:51.188 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.258 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:51.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:51.358 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:51.385 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:28:51.408 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:51.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:51.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:51.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:51.511 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:51.521 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 118241 objects 2024-09-14T04:28:51.521 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :131.712 2024-09-14T04:28:51.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241911 12.5 12:a1f1d1cb:::benchmark_data_smithi001_255951_object96251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.267407+0000", 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 411.27980406500001, 2024-09-14T04:28:51.568 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8163235549999999, 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241911 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.569 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267407+0000", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267407+0000", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267409+0000", 2024-09-14T04:28:51.570 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0640000000000001e-06 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267415+0000", 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6790000000000002e-06 2024-09-14T04:28:51.571 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267428+0000", 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3499999999999999e-05 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.572 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.267436+0000", 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7470000000000005e-06 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.980400+0000", 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.712964241 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.573 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.986254+0000", 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058538289999999996 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.030147+0000", 2024-09-14T04:28:51.574 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043892484000000002 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.037451+0000", 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073045899999999997 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.997230+0000", 2024-09-14T04:28:51.575 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95977838699999996 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.002333+0000", 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051034000000000001 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.576 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.012174+0000", 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0098413319999999995 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.053724+0000", 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041549544000000001 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.577 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.055649+0000", 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0019247839999999999 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.058115+0000", 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024668810000000002 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.578 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.062012+0000", 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038960459999999998 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.065337+0000", 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003325798 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.083731+0000", 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018393249 2024-09-14T04:28:51.579 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241928 12.5 12:a6970ac4:::benchmark_data_smithi001_255951_object96268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.326634+0000", 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 411.220576607, 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8867353179999999, 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.580 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.581 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.581 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.581 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241928 2024-09-14T04:28:51.581 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.582 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.582 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.582 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.582 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326634+0000", 2024-09-14T04:28:51.582 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326634+0000", 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.583 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326636+0000", 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.609e-06 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326641+0000", 2024-09-14T04:28:51.584 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3909999999999997e-06 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326656+0000", 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4499e-05 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.585 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.326663+0000", 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3679999999999996e-06 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.987059+0000", 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.66039556700000002 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.586 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.993160+0000", 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0061012899999999997 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.093806+0000", 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10064553499999999 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100103+0000", 2024-09-14T04:28:51.587 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062968779999999997 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.152306+0000", 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.052203271 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.588 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.157618+0000", 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053120399999999996 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.167337+0000", 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00971904 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.195105+0000", 2024-09-14T04:28:51.589 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.02776822 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.196616+0000", 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015111549999999999 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.198573+0000", 2024-09-14T04:28:51.590 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0019569639999999998 2024-09-14T04:28:51.591 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.591 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.591 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.591 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.200697+0000", 2024-09-14T04:28:51.591 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002123471 2024-09-14T04:28:51.592 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.592 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.203598+0000", 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002901299 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.213370+0000", 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097717210000000006 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.593 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241948 12.5 12:a19813e7:::benchmark_data_smithi001_255951_object96288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.477655+0000", 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 411.06955592100002, 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.71694152, 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.594 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241948 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477655+0000", 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.595 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477655+0000", 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477657+0000", 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6390000000000001e-06 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.596 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477664+0000", 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9600000000000003e-06 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477676+0000", 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2359e-05 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.597 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.477684+0000", 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0539999999999995e-06 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.993833+0000", 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.51614850899999998 2024-09-14T04:28:51.598 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.001170+0000", 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073374850000000004 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.061656+0000", 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.060485639000000001 2024-09-14T04:28:51.599 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.067487+0000", 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005830946 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.103943+0000", 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0364560009999999 2024-09-14T04:28:51.600 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.111642+0000", 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0076994949999999998 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.122567+0000", 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010925021 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.601 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.176939+0000", 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.054371743 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.177916+0000", 2024-09-14T04:28:51.602 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00097714099999999995 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.179396+0000", 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014798190000000001 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.181444+0000", 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020485030000000001 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.603 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.184409+0000", 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002964515 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.194597+0000", 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010187691 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241961 12.5 12:a773266a:::benchmark_data_smithi001_255951_object96301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.604 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.555696+0000", 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.99151479599999, 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7983760200000001, 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241961 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.605 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555696+0000", 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555696+0000", 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555698+0000", 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.669e-06 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.606 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555712+0000", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.428e-05 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555729+0000", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6816e-05 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.555738+0000", 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1689999999999998e-06 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.607 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100797+0000", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.54505842699999996 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.106699+0000", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059027259999999996 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.140659+0000", 2024-09-14T04:28:51.608 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.033959147000000002 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.146942+0000", 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0062838829999999997 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.249453+0000", 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1025104189999999 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.609 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.255065+0000", 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056125059999999997 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.610 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.264837+0000", 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0097717719999999997 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.305462+0000", 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.040625217999999998 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.307102+0000", 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0016393320000000001 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.611 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.310822+0000", 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003720152 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.339001+0000", 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.028179177 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.612 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.342392+0000", 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003390555 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.354072+0000", 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011680772000000001 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.613 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241983 12.5 12:aad29e09:::benchmark_data_smithi001_255951_object96323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.621740+0000", 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.92547141699998, 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8607624810000001, 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241983 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.614 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621740+0000", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621740+0000", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621741+0000", 2024-09-14T04:28:51.615 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.291e-06 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621746+0000", 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.259e-06 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621760+0000", 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3726999999999999e-05 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.616 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.621767+0000", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9940000000000003e-06 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.108952+0000", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.48718524099999999 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.114849+0000", 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058971450000000003 2024-09-14T04:28:51.617 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.176423+0000", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.061573456999999998 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.185447+0000", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.009024532 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.389273+0000", 2024-09-14T04:28:51.618 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2038255979999999 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.393821+0000", 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0045484480000000001 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.402813+0000", 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089915480000000002 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.619 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.456150+0000", 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.053337203 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.460365+0000", 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042153470000000004 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.464084+0000", 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037181940000000002 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.620 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.465507+0000", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014235700000000001 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.468347+0000", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028398870000000001 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.482502+0000", 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014155040000000001 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.621 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:241996 12.5 12:a2a604ee:::benchmark_data_smithi001_255951_object96336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.684112+0000", 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.863098654, 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.899881181, 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.622 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 241996 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684112+0000", 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684112+0000", 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.623 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684114+0000", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7600000000000001e-06 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684120+0000", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4399999999999996e-06 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684134+0000", 2024-09-14T04:28:51.624 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3886000000000001e-05 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.684141+0000", 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6350000000000006e-06 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.186262+0000", 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.50212099399999999 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.625 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.196725+0000", 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010462874 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.239194+0000", 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.042469162999999997 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.246178+0000", 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006983965 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.626 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.506624+0000", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.260445467 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.511861+0000", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052378370000000004 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.521420+0000", 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0095584539999999992 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.627 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.559289+0000", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.037869331999999999 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.560748+0000", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014589360000000001 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.562750+0000", 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020022939999999999 2024-09-14T04:28:51.628 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.629 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.629 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.629 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.566560+0000", 2024-09-14T04:28:51.629 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038090250000000002 2024-09-14T04:28:51.629 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.570441+0000", 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003881355 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.632 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.583994+0000", 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013552764 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242034 12.5 12:adef20fb:::benchmark_data_smithi001_255951_object96374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.849588+0000", 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.697623303, 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9540186099999999, 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.633 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242034 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849588+0000", 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.634 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849588+0000", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849589+0000", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5120000000000001e-06 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849594+0000", 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0939999999999998e-06 2024-09-14T04:28:51.635 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849606+0000", 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1579999999999999e-05 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.849613+0000", 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9120000000000001e-06 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.636 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.198951+0000", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.34933785899999997 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.213997+0000", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015046 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.271756+0000", 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.057759284000000001 2024-09-14T04:28:51.637 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.277753+0000", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059965269999999998 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.751797+0000", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.47404424 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.756273+0000", 2024-09-14T04:28:51.638 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044765650000000001 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.765148+0000", 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088745360000000006 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.775925+0000", 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010777234 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.639 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.779448+0000", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035230550000000002 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.782293+0000", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028447260000000001 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.784109+0000", 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001815711 2024-09-14T04:28:51.640 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.787742+0000", 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036330619999999998 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.803606+0000", 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015864712999999999 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.641 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242060 12.5 12:afc8509b:::benchmark_data_smithi001_255951_object96400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:00.925605+0000", 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.62160636300001, 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.945495505, 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.642 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242060 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925605+0000", 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925605+0000", 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.643 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925606+0000", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4449999999999999e-06 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925611+0000", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5950000000000001e-06 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925622+0000", 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1138000000000001e-05 2024-09-14T04:28:51.644 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:00.925628+0000", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9379999999999997e-06 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.279960+0000", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.35433204800000001 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.285770+0000", 2024-09-14T04:28:51.645 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058105370000000002 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.326054+0000", 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.040283121999999998 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.332421+0000", 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0063677780000000002 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.646 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.820129+0000", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4877075419999999 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.825067+0000", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004938568 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.834110+0000", 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0090427809999999997 2024-09-14T04:28:51.647 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.851223+0000", 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017113169000000001 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.852318+0000", 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001094275 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.648 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.854077+0000", 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00175954 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.856321+0000", 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022441499999999999 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.859461+0000", 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031400070000000002 2024-09-14T04:28:51.649 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.871100+0000", 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011638872 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242076 12.5 12:a9d525f4:::benchmark_data_smithi001_255951_object96416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:01.100591+0000", 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.44661994299997, 2024-09-14T04:28:51.650 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.870912702, 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242076 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100591+0000", 2024-09-14T04:28:51.651 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100591+0000", 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100593+0000", 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.437e-06 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.652 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100598+0000", 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5389999999999998e-06 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100609+0000", 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0903999999999999e-05 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.100616+0000", 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0589999999999997e-06 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.653 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.286319+0000", 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.18570262700000001 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.654 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:51.656 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.292464+0000", 2024-09-14T04:28:51.656 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0061450960000000001 2024-09-14T04:28:51.656 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.656 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.656 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.357154+0000", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.064689808000000001 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.362993+0000", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058395670000000004 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.919100+0000", 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5561070529999999 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.657 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.923595+0000", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044948180000000003 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.932249+0000", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0086536919999999993 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.949556+0000", 2024-09-14T04:28:51.658 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.017307154000000002 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.951888+0000", 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023319320000000001 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.954369+0000", 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024807900000000001 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.659 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.955665+0000", 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012966690000000001 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.958510+0000", 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028451409999999998 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:02.971504+0000", 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012993416000000001 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.660 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242083 12.5 12:a1698c8b:::benchmark_data_smithi001_255951_object96423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:01.217107+0000", 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 410.33010453200001, 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.856046195, 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.661 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242083 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217107+0000", 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217107+0000", 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.662 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217108+0000", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.356e-06 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217114+0000", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3640000000000004e-06 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217126+0000", 2024-09-14T04:28:51.663 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.218e-05 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.217133+0000", 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.139e-06 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.294745+0000", 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.077611924999999998 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.664 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.300623+0000", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058788449999999997 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.390210+0000", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.089586391000000001 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:01.396180+0000", 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059701989999999998 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.665 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.014207+0000", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6180270559999999 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.018648+0000", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004441225 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.027601+0000", 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0089525400000000002 2024-09-14T04:28:51.666 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.052262+0000", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.024661438000000001 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.055051+0000", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027883640000000002 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.056216+0000", 2024-09-14T04:28:51.667 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0011650289999999999 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.058095+0000", 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0018795559999999999 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.061341+0000", 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032454820000000001 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.668 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:03.073153+0000", 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011812105999999999 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242887 12.5 12:a05745ca:::benchmark_data_smithi001_255951_object97227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:05.916895+0000", 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 405.63031601199998, 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7655464320000001, 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.669 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242887 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916895+0000", 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.670 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916895+0000", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916897+0000", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7060000000000001e-06 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916903+0000", 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.161e-06 2024-09-14T04:28:51.671 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916919+0000", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5940999999999999e-05 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:05.916928+0000", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.9330000000000007e-06 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.024782+0000", 2024-09-14T04:28:51.672 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.107854319 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.035608+0000", 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010825832000000001 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.124767+0000", 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.089159246999999997 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.673 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.131336+0000", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0065688789999999997 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.613289+0000", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.481952532 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.617610+0000", 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043211509999999996 2024-09-14T04:28:51.674 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.626380+0000", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087706969999999992 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.640643+0000", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014262511 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.642530+0000", 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0018866460000000001 2024-09-14T04:28:51.675 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.644642+0000", 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021126949999999999 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.670731+0000", 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.026088626 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.676 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.673071+0000", 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023399240000000002 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.682442+0000", 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0093706320000000003 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:242914 12.5 12:a83d8079:::benchmark_data_smithi001_255951_object97254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-09-14T04:28:51.677 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:06.226686+0000", 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 405.32052463799999, 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7581916020000001, 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 242914 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.678 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226686+0000", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226686+0000", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226688+0000", 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3599999999999999e-06 2024-09-14T04:28:51.679 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226693+0000", 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2950000000000001e-06 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226706+0000", 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2437e-05 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.680 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.226712+0000", 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4049999999999997e-06 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.365458+0000", 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.138745915 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.372853+0000", 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073948440000000002 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.681 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.471867+0000", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.099014501000000005 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:06.477564+0000", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056963990000000004 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.819761+0000", 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.342197759 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.682 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.824191+0000", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044300540000000001 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.832940+0000", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.008748591 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.874370+0000", 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041430157000000002 2024-09-14T04:28:51.683 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.876342+0000", 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0019721750000000001 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.878313+0000", 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001970669 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.684 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.972576+0000", 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.094262995000000002 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.975012+0000", 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002436148 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:07.984878+0000", 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0098658979999999997 2024-09-14T04:28:51.685 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243978 12.5 12:a67ec102:::benchmark_data_smithi001_255951_object98318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:13.756994+0000", 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 397.79021756999998, 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.78058691, 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.686 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243978 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.756994+0000", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.756994+0000", 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.687 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.756994+0000", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.02e-07 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.756997+0000", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9560000000000002e-06 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.757006+0000", 2024-09-14T04:28:51.688 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6270000000000006e-06 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.757012+0000", 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2550000000000003e-06 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.651505+0000", 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.89449272499999999 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.689 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.658012+0000", 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006506721 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.723475+0000", 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.065463246000000003 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.730724+0000", 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0072492549999999996 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.690 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.444780+0000", 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.71405554000000004 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.450489+0000", 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057093619999999999 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.691 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.461660+0000", 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011170906 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.514696+0000", 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.053036089000000002 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.516580+0000", 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001883855 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.692 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.518803+0000", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0022233880000000002 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.521491+0000", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0026875290000000001 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.524981+0000", 2024-09-14T04:28:51.693 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034902900000000001 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.537580+0000", 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012599264000000001 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:243997 12.5 12:a6e1752e:::benchmark_data_smithi001_255951_object98337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:13.887925+0000", 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 397.65928609299999, 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.817843243, 2024-09-14T04:28:51.694 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 243997 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887925+0000", 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.695 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887925+0000", 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887926+0000", 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.223e-06 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.696 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887931+0000", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0810000000000001e-06 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887944+0000", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2368000000000001e-05 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.887951+0000", 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1439999999999997e-06 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.697 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.741328+0000", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.85337694900000005 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.748790+0000", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0074625489999999997 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.821221+0000", 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.072430321000000006 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.698 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.830260+0000", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0090396079999999993 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.588168+0000", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.75790727899999999 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.595103+0000", 2024-09-14T04:28:51.699 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0069351830000000001 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.609742+0000", 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014639635 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.667544+0000", 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.057801445999999999 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.700 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.670643+0000", 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0030996230000000001 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.673351+0000", 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002707685 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.676223+0000", 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028716280000000002 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.701 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.679951+0000", 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037285949999999999 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.705768+0000", 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.025816926 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.702 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:244032 12.5 12:af505557:::benchmark_data_smithi001_255951_object98372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:13.997327+0000", 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 397.54988362, 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.745359828, 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 244032 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.703 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997327+0000", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997327+0000", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997329+0000", 2024-09-14T04:28:51.704 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4780000000000001e-06 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997335+0000", 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8289999999999996e-06 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997353+0000", 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7802e-05 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.705 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:13.997361+0000", 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6649999999999992e-06 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.843447+0000", 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.84608536199999995 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.852243+0000", 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087965119999999994 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.706 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.906279+0000", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.054036275000000002 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.913112+0000", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.006832533 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.640183+0000", 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.72707075700000001 2024-09-14T04:28:51.707 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.645683+0000", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005500296 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.656119+0000", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010435722999999999 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.674441+0000", 2024-09-14T04:28:51.708 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.018321983999999999 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.709809+0000", 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.035368300999999998 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.715622+0000", 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058130589999999998 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.709 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.721673+0000", 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0060511920000000004 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.727609+0000", 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059356010000000004 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.742687+0000", 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015078459000000001 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.710 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:244089 12.5 12:a2105d2b:::benchmark_data_smithi001_255951_object98429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:14.183771+0000", 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 397.36344000999998, 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8061146770000001, 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:28:51.711 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 244089 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183771+0000", 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183771+0000", 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.712 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183772+0000", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.246e-06 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183782+0000", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1740000000000006e-06 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183797+0000", 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5301000000000001e-05 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.713 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.183804+0000", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0350000000000004e-06 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.863197+0000", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.67939312299999999 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for missing object", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.870590+0000", 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073934220000000002 2024-09-14T04:28:51.714 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.940514+0000", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.069923938000000005 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for rw locks", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:14.947410+0000", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0068958509999999997 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.896530+0000", 2024-09-14T04:28:51.715 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94911954499999995 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.902321+0000", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057916540000000002 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,3,6", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.916224+0000", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013902786 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.957263+0000", 2024-09-14T04:28:51.716 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.041038531000000003 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.960713+0000", 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003450505 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.964354+0000", 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036412800000000002 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.717 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.965732+0000", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0013774950000000001 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.969279+0000", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035472590000000001 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:15.989886+0000", 2024-09-14T04:28:51.718 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.020606532 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216048 13.8 13:17200947:::benchmark_data_smithi001_306460_object97804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.734695+0000", 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.812516436999999, 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1316547809999999, 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.719 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216048 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734695+0000", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734695+0000", 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.720 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734696+0000", 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.734e-06 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734702+0000", 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3940000000000004e-06 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.721 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734722+0000", 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9834999999999999e-05 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734730+0000", 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5059999999999995e-06 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734836+0000", 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000105987 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.722 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.734847+0000", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1167e-05 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256243+0000", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.521396097 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256256+0000", 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2663000000000001e-05 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.723 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.853383+0000", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.597127043 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.853507+0000", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012438 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.853997+0000", 2024-09-14T04:28:51.724 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048979799999999995 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865848+0000", 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011851083 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865863+0000", 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4929000000000001e-05 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.725 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865952+0000", 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8280000000000002e-05 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865995+0000", 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3708000000000002e-05 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866095+0000", 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000100239 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.726 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866349+0000", 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025393799999999999 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216056 13.8 13:13a911b7:::benchmark_data_smithi001_306460_object97812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.740225+0000", 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.806985791000002, 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1265233280000002, 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.727 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216056 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740225+0000", 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.728 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740225+0000", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740226+0000", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.141e-06 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740231+0000", 2024-09-14T04:28:51.729 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4469999999999999e-06 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740245+0000", 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4453e-05 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740253+0000", 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5780000000000001e-06 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.730 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740314+0000", 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0599e-05 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.740322+0000", 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5059999999999995e-06 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256281+0000", 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.51595877099999998 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.731 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256289+0000", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0020000000000006e-06 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.856627+0000", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.600338034 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.856728+0000", 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000100894 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.732 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.857157+0000", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00042895800000000001 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865898+0000", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0087417840000000007 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865912+0000", 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3662000000000001e-05 2024-09-14T04:28:51.733 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866466+0000", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00055367100000000005 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866513+0000", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7593999999999999e-05 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866638+0000", 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012435899999999999 2024-09-14T04:28:51.734 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866749+0000", 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000110875 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216064 13.8 13:15380050:::benchmark_data_smithi001_306460_object97820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.747386+0000", 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.799825392999999, 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1196474460000001, 2024-09-14T04:28:51.735 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216064 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747386+0000", 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.736 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747386+0000", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747386+0000", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7899999999999997e-07 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747391+0000", 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0550000000000001e-06 2024-09-14T04:28:51.737 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747405+0000", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4083000000000001e-05 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747412+0000", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9160000000000004e-06 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747472+0000", 2024-09-14T04:28:51.738 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0294000000000002e-05 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747480+0000", 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4980000000000006e-06 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256312+0000", 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.508831531 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.739 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256320+0000", 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8120000000000008e-06 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.859573+0000", 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6032538220000001 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.859727+0000", 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000153533 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.740 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.860161+0000", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043423700000000002 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.865917+0000", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057558569999999996 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866410+0000", 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00049260099999999995 2024-09-14T04:28:51.741 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866797+0000", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038768299999999999 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866855+0000", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7181e-05 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.866925+0000", 2024-09-14T04:28:51.742 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0327999999999995e-05 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.867033+0000", 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000108236 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216065 13.8 13:153fb362:::benchmark_data_smithi001_306460_object97821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.747694+0000", 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 32.799517581000003, 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1404628880000001, 2024-09-14T04:28:51.743 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216065 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747694+0000", 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.744 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747694+0000", 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747694+0000", 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6799999999999999e-07 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.745 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747698+0000", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0509999999999998e-06 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747708+0000", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.55e-06 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747714+0000", 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2230000000000004e-06 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.746 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747769+0000", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4449e-05 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.747777+0000", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.67e-06 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256343+0000", 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.50856518500000003 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.747 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.256350+0000", 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.007e-06 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.867075+0000", 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6107250450000001 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:28:51.748 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.867190+0000", 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115635 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.867704+0000", 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00051333399999999995 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.877620+0000", 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0099161330000000006 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.749 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.877668+0000", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8754000000000001e-05 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.877713+0000", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4987999999999997e-05 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.887987+0000", 2024-09-14T04:28:51.750 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010273157999999999 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.888040+0000", 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3603999999999997e-05 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.888156+0000", 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116234 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:28:51.751 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:28:51.752 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:28:51.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:51.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:51.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:28:51.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:51.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:51.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:52.003 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:52.025 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:52.127 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:52.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:52.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:52.278 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:52.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:52.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:52.407 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:52.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:52.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:52.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:52.636 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:52.658 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: pgmap v1543: 119 pgs: 119 active+clean; 138 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]} : dispatch 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]} : dispatch 2024-09-14T04:28:52.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: pgmap v1543: 119 pgs: 119 active+clean; 138 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]} : dispatch 2024-09-14T04:28:52.722 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]} : dispatch 2024-09-14T04:28:52.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:52.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:52.831 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2398459909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: pgmap v1543: 119 pgs: 119 active+clean; 138 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]} : dispatch 2024-09-14T04:28:52.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]} : dispatch 2024-09-14T04:28:52.885 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:52.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:52.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:53.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:53.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:53.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:53.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:53.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:53.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:53.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:28:53.300 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:53.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:53.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:53.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:53.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:53.542 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:28:53.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:28:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-14T04:28:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]}]': finished 2024-09-14T04:28:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2024-09-14T04:28:53.553 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[112165]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:28:53.554 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:28:53.554 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-14T04:28:53.554 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]}]': finished 2024-09-14T04:28:53.554 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2024-09-14T04:28:53.554 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:53 smithi001 ceph-mon[107399]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:28:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-14T04:28:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-14T04:28:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [3, 5]}]': finished 2024-09-14T04:28:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2024-09-14T04:28:53.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:53 smithi012 ceph-mon[101180]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:28:53.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:53.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:53.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:53.772 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:53.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.795 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:53.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:53.896 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:53.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:53.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:54.020 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:54.042 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:54.144 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:54.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.168 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:54.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:54.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:54.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:54.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:54.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:54.541 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:54.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:28:54.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:54.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:54.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:54 smithi001 ceph-mon[107399]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:28:54.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:54 smithi001 ceph-mon[107399]: pgmap v1546: 119 pgs: 1 peering, 118 active+clean; 82 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-14T04:28:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:54 smithi001 ceph-mon[112165]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:28:54.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:54 smithi001 ceph-mon[112165]: pgmap v1546: 119 pgs: 1 peering, 118 active+clean; 82 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-14T04:28:54.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:54.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:54.820 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:54 smithi012 ceph-mon[101180]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:28:54.820 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:54 smithi012 ceph-mon[101180]: pgmap v1546: 119 pgs: 1 peering, 118 active+clean; 82 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-14T04:28:54.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:54.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:54.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:55.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:55.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:55.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:55.163 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:55.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:55.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:55.331 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:28:55.331 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:28:55.331 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2024-09-14T04:28:55.331 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:28:55.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:55.412 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:55.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:55.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:55.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:55.666 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:55.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[107399]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:28:55.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[107399]: 13.19 starting backfill to osd.3 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[107399]: 13.19 starting backfill to osd.6 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.671 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[107399]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.671 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[112165]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:28:55.672 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[112165]: 13.19 starting backfill to osd.3 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.672 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[112165]: 13.19 starting backfill to osd.6 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.672 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:55 smithi001 ceph-mon[112165]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.744 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:55 smithi012 ceph-mon[101180]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:28:55.744 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:55 smithi012 ceph-mon[101180]: 13.19 starting backfill to osd.3 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.744 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:55 smithi012 ceph-mon[101180]: 13.19 starting backfill to osd.6 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.744 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:55 smithi012 ceph-mon[101180]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 484'11780 2024-09-14T04:28:55.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:55.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:55.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:55.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:55.923 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:28:56.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:28:56.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:28:56.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:28:56.142 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:56.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:56.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:56.261 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:28:56.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:56.288 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:56.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:56.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:56.432 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 13 pgp_num to 26 2024-09-14T04:28:56.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:56.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:56.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:56.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[112165]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:28:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[112165]: pgmap v1549: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 2.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2024-09-14T04:28:56.702 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2021411278' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.703 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[107399]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:28:56.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[107399]: pgmap v1549: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 2.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2024-09-14T04:28:56.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2021411278' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.703 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:56.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:56 smithi012 ceph-mon[101180]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:28:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:56 smithi012 ceph-mon[101180]: pgmap v1549: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 2.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2024-09-14T04:28:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2021411278' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:56 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"} : dispatch 2024-09-14T04:28:56.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:56.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:56.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:28:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:28:56] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-14T04:28:57.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:57.040 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:57.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:57.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:57.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:57.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:57.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:57.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:57.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:57.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:57.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:57.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:57.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:57 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:28:57.688 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:57 smithi001 ceph-mon[112165]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:28:57.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:57 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:28:57.688 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:57 smithi001 ceph-mon[107399]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:28:57.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:57.792 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:57 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2024-09-14T04:28:57.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:57 smithi012 ceph-mon[101180]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:28:57.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:28:57.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:57.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:28:58.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:58.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:58.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:28:58.169 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:28:58.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:58.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:58.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:58.419 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:58.459 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:28:58.459 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:28:58.459 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:28:58.460 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:28:58.476 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:28:58.476 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:28:58.492 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:28:58.493 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:28:58.509 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:28:58.510 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:28:58.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:58.526 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:28:58.526 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:28:58.544 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:28:58.544 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:28:58.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:58.560 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:28:58.560 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:28:58.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:28:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.673 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:58.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:28:58.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:58 smithi001 ceph-mon[107399]: pgmap v1551: 119 pgs: 1 peering, 118 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2024-09-14T04:28:58.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1164569515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:58 smithi001 ceph-mon[112165]: pgmap v1551: 119 pgs: 1 peering, 118 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2024-09-14T04:28:58.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1164569515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:58.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:58.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:58.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:58.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:58 smithi012 ceph-mon[101180]: pgmap v1551: 119 pgs: 1 peering, 118 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2024-09-14T04:28:58.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1164569515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:28:58.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:58.924 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:58.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:58.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:59.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:28:59.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:28:59.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:59.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:59.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:59.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:28:59.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:59.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:59.441 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:28:59.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:28:59.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:28:59.568 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:28:59.569 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:28:59.569 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:28:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:59.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:59.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:28:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:28:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:28:59.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:28:59.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:28:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:28:59.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:28:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:28:59.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:28:59.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:00.027 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:00.050 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:00.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:29:00.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:00.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:00.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:00.323 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:00.338 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:29:00.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:00.452 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:00.554 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:00.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:00.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:00.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:00.703 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[107399]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[107399]: pgmap v1553: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 173 op/s 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[112165]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:29:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:00 smithi001 ceph-mon[112165]: pgmap v1553: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 173 op/s 2024-09-14T04:29:00.746 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:00 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:00.746 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1496418405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:29:00.746 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:00 smithi012 ceph-mon[101180]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:29:00.746 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:00 smithi012 ceph-mon[101180]: pgmap v1553: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 173 op/s 2024-09-14T04:29:00.764 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2024-09-14T04:29:00.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2024-09-14T04:29:00.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:00.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:00.933 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:00.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:00.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:00.978 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:01.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:01.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:01.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:01.183 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:01.206 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:01.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:01.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:01.433 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:01.455 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:01.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:01.580 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:01.677 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:29:01.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:01.705 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:01.807 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:01.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:01.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:01.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:01.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:01.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:02.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:02.059 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:02.079 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:02.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:02.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:02.279 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:29:02.279 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:29:02.295 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:29:02.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.328 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:02.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:02.429 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.452 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:02.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:02.553 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:02.576 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:02.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:02.700 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:02.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:02.825 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:02.873 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:02 smithi001 ceph-mon[107399]: pgmap v1554: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2024-09-14T04:29:02.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:02 smithi001 ceph-mon[112165]: pgmap v1554: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2024-09-14T04:29:02.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:02 smithi012 ceph-mon[101180]: pgmap v1554: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2024-09-14T04:29:02.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:29:02.927 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:02.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:02.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:03.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:03.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:03.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:03.235 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:03.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:03.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:03.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:03.487 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:03.589 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:03.612 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:03.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:03.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:03.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:03.868 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:03.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:03.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:03.993 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:03.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:04.095 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:04.118 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:04.217 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:04.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:04.242 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:04.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:04.366 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:04.468 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:04.492 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:04.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:04.618 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:04.632 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_8' removed 2024-09-14T04:29:04.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:04.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:04.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:04.869 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:04.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[107399]: pgmap v1555: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T04:29:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1902712296' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[112165]: pgmap v1555: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T04:29:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1902712296' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:04 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:04.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:04.994 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:04.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:04.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:04 smithi012 ceph-mon[101180]: pgmap v1555: 119 pgs: 119 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-14T04:29:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:04 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1902712296' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:04 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:29:05.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:05.119 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:05.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:05.250 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:05.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:05.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:05.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:05.501 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:05.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:05.625 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:05.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:05.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:05.765 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 7, 6] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:29:05.765 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:29:05.765 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:29:05.765 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2024-09-14T04:29:05.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 0 2024-09-14T04:29:05.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:05.873 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:05.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:05.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[107399]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[112165]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:29:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:05.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:06.000 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:05 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:29:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:05 smithi012 ceph-mon[101180]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:29:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:06.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:06.124 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:06.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:06.131 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:29:06.131 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:29:06.152 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:29:06.152 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:29:06.168 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:29:06.169 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:29:06.185 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:29:06.186 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:29:06.203 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:29:06.203 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:29:06.220 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:29:06.220 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:29:06.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:06.236 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:29:06.237 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:29:06.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:06.253 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:29:06.253 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:29:06.353 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:06.379 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:06.481 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:06.504 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:06.606 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.629 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:06.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:06.730 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.754 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:06.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:06.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:06.883 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:06.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:06.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:06 smithi001 ceph-mon[112165]: pgmap v1557: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/856911679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:06 smithi001 ceph-mon[107399]: pgmap v1557: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/856911679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:06] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.51.0" 2024-09-14T04:29:06.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:07.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:06 smithi012 ceph-mon[101180]: pgmap v1557: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/856911679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:07.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:07.138 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:07.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:07.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:07.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:07.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:07.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:07.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:07.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:07.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:07.655 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:07.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:07.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:07.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:07.887 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:07.911 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:07.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:07.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:08.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:08.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:08.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:08.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:08.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:08.271 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:08.295 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:08.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:08.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:08.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.550 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:08.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:08.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:08.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:08.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:08.806 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:08.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:08.898 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:08 smithi012 ceph-mon[101180]: pgmap v1558: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:08.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:08.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:08.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:08.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:08 smithi001 ceph-mon[107399]: pgmap v1558: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:08.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:08 smithi001 ceph-mon[112165]: pgmap v1558: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:09.038 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:09.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:09.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:09.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:09.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:09.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:09.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:09.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:09.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:09.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:09.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:09.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:09.803 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:09.829 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:09.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:09.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:09.957 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:09.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:10.059 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:10.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:10.186 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:10.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:10.313 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:10.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:10.395 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:29:10.396 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:29:10.396 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2024-09-14T04:29:10.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_9 16 2024-09-14T04:29:10.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.465 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:10.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:10.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:10.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:10.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:10.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:10.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:10.838 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:10.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:10.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:10.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:10.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:10 smithi001 ceph-mon[107399]: pgmap v1559: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:10 smithi001 ceph-mon[112165]: pgmap v1559: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:11.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:11.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:11.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:10 smithi012 ceph-mon[101180]: pgmap v1559: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:11.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:11.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:11.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:11.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:11.441 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:11.467 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:11.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:11.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:11.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:11.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:11.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:11.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:11.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:11.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:11.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:11.998 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:29:11.999 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:29:11.999 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:29:11.999 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:29:12.079 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.105 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:12.189 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:12.206 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:12.232 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:12.278 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:12.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:12.357 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:12.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:12.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:12.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:12.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:12.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:12.653 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:29:12.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:12.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:12.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:12.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:12.947 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[107399]: pgmap v1560: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:12.947 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1073793750' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:12.947 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:12.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[112165]: pgmap v1560: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:12.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1073793750' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:12.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:12.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:29:12.970 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:12.991 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:12.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:13.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:29:13.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:12 smithi012 ceph-mon[101180]: pgmap v1560: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1073793750' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:13.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:13.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:13.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:13.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:13.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:13.383 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:13.485 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:13.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:13.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:13.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:13.741 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:13.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:13.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:13.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:29:13.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:29:13.865 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:13.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[107399]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:29:13.866 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:29:13.866 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:29:13.866 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:13.867 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:13 smithi001 ceph-mon[112165]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:29:13.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:13.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:13.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:13.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:14.024 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:14.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:13 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:29:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:13 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:29:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:13 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:13 smithi012 ceph-mon[101180]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:29:14.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.154 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:14.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:14.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:14.283 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:14.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:14.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:14.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:14.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:14.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:14.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:14.759 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:14.781 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:14.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:14.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:14.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:14.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:14 smithi001 ceph-mon[107399]: pgmap v1562: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:14 smithi001 ceph-mon[112165]: pgmap v1562: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:15.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:15.030 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:14 smithi012 ceph-mon[101180]: pgmap v1562: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:15.132 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:15.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:15.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:15.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:15.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:15.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:15.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:15.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:15.646 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:15.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:15.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:15.799 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:15.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:15.927 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:15.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:16.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:16.056 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:16.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:16.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:16.170 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:16.181 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:16.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:16.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:16.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:16.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:16.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:16.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:16.536 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.560 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:16.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:16.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:16.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:16.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:16 smithi001 ceph-mon[107399]: pgmap v1563: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:16.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:16 smithi001 ceph-mon[112165]: pgmap v1563: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:16.721 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:16] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.51.0" 2024-09-14T04:29:16.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:16.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:16.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:16.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:16.939 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:16.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:16.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:17.040 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:17.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:16 smithi012 ceph-mon[101180]: pgmap v1563: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:17.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:17.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:29:17.188 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:17.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:17.312 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:17.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:17.414 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:17.437 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:17.539 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:17.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:17.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:17.664 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.5 primary-affinity to 0 (802) 2024-09-14T04:29:17.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:17.765 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:17.765 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4055120661' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:17.765 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:17.766 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:17.766 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4055120661' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:17.766 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:17 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:17.794 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:17.817 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:17.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:17.941 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:17.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:18.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:18.066 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:18.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:17 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4055120661' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:17 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-14T04:29:18.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:18.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:18.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:18.314 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:18.416 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:18.438 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:18.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:18.563 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:18.662 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_9' created 2024-09-14T04:29:18.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:18.693 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:18.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:18.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:18.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:18.942 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:18.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:18.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: pgmap v1564: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/28038238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[112165]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: pgmap v1564: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:18.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/28038238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-14T04:29:18.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:18 smithi001 ceph-mon[107399]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:29:18.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: pgmap v1564: 93 pgs: 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/28038238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-14T04:29:19.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:18 smithi012 ceph-mon[101180]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:29:19.045 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:19.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:19.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:19.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:19.294 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:19.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:19.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:19.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:19.543 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:19.565 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:19.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:19.696 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:19.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:19.798 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:19.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:19.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:19.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:19.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:19.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:19.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[112165]: 1.1 scrub starts 2024-09-14T04:29:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[112165]: 1.1 scrub ok 2024-09-14T04:29:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[112165]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:29:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[107399]: 1.1 scrub starts 2024-09-14T04:29:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[107399]: 1.1 scrub ok 2024-09-14T04:29:19.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:19 smithi001 ceph-mon[107399]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:29:20.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:20.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:19 smithi012 ceph-mon[101180]: 1.1 scrub starts 2024-09-14T04:29:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:19 smithi012 ceph-mon[101180]: 1.1 scrub ok 2024-09-14T04:29:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:19 smithi012 ceph-mon[101180]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:29:20.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:20.194 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:20.296 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:20.319 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:20.421 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:20.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:20.544 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:20.567 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:20.607 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2024-09-14T04:29:20.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:20.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:20.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:20.815 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:20.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:20.940 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:20.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: pgmap v1567: 109 pgs: 1 creating+peering, 15 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[107399]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:29:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:29:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: pgmap v1567: 109 pgs: 1 creating+peering, 15 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:29:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:20 smithi001 ceph-mon[112165]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:29:21.042 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:21.065 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:29:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:29:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: pgmap v1567: 109 pgs: 1 creating+peering, 15 unknown, 93 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:29:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:21.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:20 smithi012 ceph-mon[101180]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:29:21.126 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:21.168 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:21.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:21.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:21.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:21.419 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:21.445 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:21.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:21.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:21.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:21.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:21.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:21.709 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:29:21.709 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:29:21.709 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:29:21.709 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:29:21.709 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:29:21.796 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:21.819 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:21.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:21.944 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:21.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:22.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:22.070 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:22.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:22.195 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:22.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:22.321 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:22 smithi012 ceph-mon[101180]: pgmap v1570: 109 pgs: 7 creating+peering, 3 unknown, 99 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:22.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[112165]: pgmap v1570: 109 pgs: 7 creating+peering, 3 unknown, 99 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:22.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:22.357 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:22 smithi001 ceph-mon[107399]: pgmap v1570: 109 pgs: 7 creating+peering, 3 unknown, 99 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:22.381 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:29:22.423 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:22.447 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:22.548 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:22.570 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:22.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:22.698 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:22.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:22.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:22.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:22.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:22.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:23.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:23.084 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:23.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.213 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:23.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:23.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:23.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:23.442 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:23.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:23.537 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:23 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3174946220' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:23.538 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:23 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3174946220' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:23.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:23.575 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:29:23.575 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:29:23.575 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:29:23.575 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:29:23.575 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:29:23.576 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:29:23.577 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:29:23.577 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:29:23.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:23.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:23.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:23.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:23.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:23 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3174946220' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:23.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:23.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:23.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:23.973 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:23.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:24.074 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:24.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:24.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:24.225 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:24.326 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:24.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:24.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:24.548 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 7, 6] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:29:24.549 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:29:24.549 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 4, 5, 3, 7, 6] 2024-09-14T04:29:24.549 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd out 2 2024-09-14T04:29:24.571 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:24 smithi001 ceph-mon[107399]: pgmap v1571: 109 pgs: 7 creating+peering, 102 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:24.572 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:24 smithi001 ceph-mon[112165]: pgmap v1571: 109 pgs: 7 creating+peering, 102 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:24.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:24.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:24.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:24.835 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:24.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:24 smithi012 ceph-mon[101180]: pgmap v1571: 109 pgs: 7 creating+peering, 102 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:24.857 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:24.959 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:24.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:24.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:25.084 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:25.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:25.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:25.235 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:25.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:25.297 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:25.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.362 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:25.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:25.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:25.489 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:25.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:25.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:25.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:25.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:25.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:25.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:25.866 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:25.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:25.991 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:25.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:26.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:26.115 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:26.217 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:26.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:26.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:29:26.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:26.365 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:26.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:26.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:26.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:26.623 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:26.724 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.748 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:26.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:26.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:26.877 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:26.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:26 smithi001 ceph-mon[107399]: pgmap v1572: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:26 smithi001 ceph-mon[112165]: pgmap v1572: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:26] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:29:26.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:27.005 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:27.007 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:26 smithi012 ceph-mon[101180]: pgmap v1572: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:27.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:27.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:27.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:27.262 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:27.290 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:27.364 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:27.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:27.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:27.515 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:27.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:27.645 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:27.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:27.773 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:27.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:27.900 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:27.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:28.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:28.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:28.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:28.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:28.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:28.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:28.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:28.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:28.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:28.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:28.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:28.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:28.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:28.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:28.665 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:28.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:28.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:28.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:28.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:28.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:28.969 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:28 smithi001 ceph-mon[107399]: pgmap v1573: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:28.969 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:28 smithi001 ceph-mon[112165]: pgmap v1573: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:29.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:29.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:28 smithi012 ceph-mon[101180]: pgmap v1573: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:29.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:29.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:29.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:29.299 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:29.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:29.422 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:29.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:29.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:29.643 INFO:teuthology.orchestra.run.smithi001.stderr:enabled application 'rados' on pool 'unique_pool_9' 2024-09-14T04:29:29.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:29.674 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:29.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:29.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:29.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:29.902 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:29.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:29.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:29.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:29 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/138513256' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:29.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:29 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:29 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/138513256' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:29 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:30.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:30.053 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:29 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/138513256' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:29 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:29:30.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:30.178 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:30.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.308 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:30.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:30.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:30.437 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:30.538 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:30.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:30.642 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:29:30.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:30.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:30.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:30.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:30.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:30.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:30.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:30.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:30.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[107399]: pgmap v1574: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[107399]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[112165]: pgmap v1574: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[112165]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:29:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:30 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:29:31.038 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:31.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:31.077 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:31.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:30 smithi012 ceph-mon[101180]: pgmap v1574: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:30 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:29:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:30 smithi012 ceph-mon[101180]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:29:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:30 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:29:31.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:31.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:31.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:31.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:31.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:31.385 DEBUG:teuthology.orchestra.run.smithi001:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2024-09-14T04:29:31.388 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:29:31.388 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:29:31.410 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:29:31.410 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:29:31.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:31.439 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:hints = 1 2024-09-14T04:29:31.439 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:29:31.439 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object prefix: benchmark_data_smithi001_337422 2024-09-14T04:29:31.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:29:31.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:29:31.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:31.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:31.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:31.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:31.756 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:31.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:31.895 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:31.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[107399]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:29:31.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:31 smithi001 ceph-mon[112165]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:29:31.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.030 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:31 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:31 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2532886075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:29:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:31 smithi012 ceph-mon[101180]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:29:32.131 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:32.179 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:32.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:32.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:32.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:32.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 1 16 3476 3460 216.238 216.25 0.00313577 0.00460992 2024-09-14T04:29:32.445 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:32.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:32.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:32.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:32.711 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:32.788 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:32 smithi001 ceph-mon[107399]: pgmap v1577: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:32.788 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:32 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:29:32.789 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:32 smithi001 ceph-mon[112165]: pgmap v1577: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:32.789 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:32 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:29:32.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:32.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:32.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:32.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:32.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:33.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:33.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:32 smithi012 ceph-mon[101180]: pgmap v1577: 109 pgs: 109 active+clean; 582 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-14T04:29:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:32 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:29:33.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:33.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:33.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:33.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:33.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:33.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:33.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 2 16 6581 6565 205.142 194.062 0.00632131 0.00486547 2024-09-14T04:29:33.497 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:33.530 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:33.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:33.670 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:33.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:33.803 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:33.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:33.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:29:33.933 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:33.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:34.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:34.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.081 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:34.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:34.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:34.210 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:34.312 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:34.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:34.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:34.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 3 16 9518 9502 197.943 183.562 0.00577539 0.00504514 2024-09-14T04:29:34.461 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:34.505 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:34.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:29:34.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:34.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:34.753 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:34.781 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:34.883 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:34.925 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:34.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:34.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:34 smithi001 ceph-mon[107399]: pgmap v1578: 109 pgs: 109 active+clean; 28 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-14T04:29:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:34 smithi001 ceph-mon[112165]: pgmap v1578: 109 pgs: 109 active+clean; 28 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-14T04:29:35.026 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:35.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:35.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:34 smithi012 ceph-mon[101180]: pgmap v1578: 109 pgs: 109 active+clean; 28 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-14T04:29:35.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:35.198 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:35.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:35.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:35.440 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 4 16 12199 12183 190.341 167.562 0.00309457 0.00524594 2024-09-14T04:29:35.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:35.491 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:35.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:35.633 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:35.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:35.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:35.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:35.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:35.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:36.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:36.092 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:36.092 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:36.092 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[107399]: pgmap v1579: 109 pgs: 109 active+clean; 563 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:29:36.093 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:36.093 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:36.093 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:35 smithi001 ceph-mon[112165]: pgmap v1579: 109 pgs: 109 active+clean; 563 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:29:36.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:36.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:36.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:36.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:36.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:35 smithi012 ceph-mon[101180]: pgmap v1579: 109 pgs: 109 active+clean; 563 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:29:36.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:36.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 5 16 14858 14842 185.508 166.188 0.00771097 0.00538302 2024-09-14T04:29:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.452 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:36.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:36.582 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:36.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:36.712 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:36.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:36.841 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:36.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:36.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:36.964 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:36] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-09-14T04:29:36.985 INFO:teuthology.orchestra.run.smithi001.stderr:marked out osd.2. 2024-09-14T04:29:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.000 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:37.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:37.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:37.124 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:37.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/649336811' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:37 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:37 smithi001 ceph-mon[112165]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-14T04:29:37.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/649336811' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:37 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:37 smithi001 ceph-mon[107399]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-14T04:29:37.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:37 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:29:36.972+0000 7fbfffcba640 -1 mon.a@0(leader).osd e498 definitely_dead 0 2024-09-14T04:29:37.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:37.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/649336811' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:36 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-14T04:29:37.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:36 smithi012 ceph-mon[101180]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-14T04:29:37.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:37.373 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:37.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 6 16 16018 16002 166.67 72.5 0.00399548 0.0055558 2024-09-14T04:29:37.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:37.497 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:37.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:37.623 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:37.725 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.749 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:37.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:37.850 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:37.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:37.878 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:37.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:37.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:38.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:38.124 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:38.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:38.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:38.261 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:38 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-14T04:29:38.261 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:38 smithi012 ceph-mon[101180]: osdmap e499: 8 total, 8 up, 7 in 2024-09-14T04:29:38.261 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3675505975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:38.261 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:38 smithi012 ceph-mon[101180]: pgmap v1581: 109 pgs: 109 active+clean; 678 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:29:38.261 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:38 smithi012 ceph-mon[101180]: osdmap e500: 8 total, 8 up, 7 in 2024-09-14T04:29:38.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:38.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:38.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:38.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 7 16 16018 16002 142.859 0 - 0.0055558 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[107399]: osdmap e499: 8 total, 8 up, 7 in 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3675505975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[107399]: pgmap v1581: 109 pgs: 109 active+clean; 678 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[107399]: osdmap e500: 8 total, 8 up, 7 in 2024-09-14T04:29:38.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-14T04:29:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[112165]: osdmap e499: 8 total, 8 up, 7 in 2024-09-14T04:29:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3675505975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:29:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[112165]: pgmap v1581: 109 pgs: 109 active+clean; 678 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:29:38.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:38 smithi001 ceph-mon[112165]: osdmap e500: 8 total, 8 up, 7 in 2024-09-14T04:29:38.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:38.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:38.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:38.655 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:38.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:38.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:38.886 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:38.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:38.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:39.014 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:39.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:39.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:39.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:39.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:39.306 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:39.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:39.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 8 16 16720 16704 130.486 21.9375 0.00948583 0.00765656 2024-09-14T04:29:39.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:39.485 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:39.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:39.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:39.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:39.751 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:39.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:39.888 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:39.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:39.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:40.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:39 smithi012 ceph-mon[101180]: osdmap e501: 8 total, 8 up, 7 in 2024-09-14T04:29:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:39 smithi012 ceph-mon[101180]: osdmap e502: 8 total, 8 up, 7 in 2024-09-14T04:29:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:39 smithi012 ceph-mon[101180]: pgmap v1585: 109 pgs: 109 active+clean; 713 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-14T04:29:40.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:40.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:40.265 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:40.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:40.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[112165]: osdmap e501: 8 total, 8 up, 7 in 2024-09-14T04:29:40.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[112165]: osdmap e502: 8 total, 8 up, 7 in 2024-09-14T04:29:40.352 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[112165]: pgmap v1585: 109 pgs: 109 active+clean; 713 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-14T04:29:40.352 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:29:40.352 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:29:40.352 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:29:40.352 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:29:40.353 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:29:40.354 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:29:40.354 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:29:40.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[107399]: osdmap e501: 8 total, 8 up, 7 in 2024-09-14T04:29:40.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[107399]: osdmap e502: 8 total, 8 up, 7 in 2024-09-14T04:29:40.354 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:39 smithi001 ceph-mon[107399]: pgmap v1585: 109 pgs: 109 active+clean; 713 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-14T04:29:40.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:40.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 9 16 18746 18730 130.056 126.625 0.00252551 0.00768064 2024-09-14T04:29:40.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:40.548 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.582 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:40.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:40.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:40.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:40.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:40.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:40.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:40.994 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:40.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:41.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.137 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:41.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:41.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:41.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:41.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:41.441 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 10 16 20673 20657 129.093 120.438 0.00367662 0.00773735 2024-09-14T04:29:41.449 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:41.450 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:41.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:41 smithi001 ceph-mon[107399]: osdmap e503: 8 total, 8 up, 7 in 2024-09-14T04:29:41.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:41 smithi001 ceph-mon[112165]: osdmap e503: 8 total, 8 up, 7 in 2024-09-14T04:29:41.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:41.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:41 smithi012 ceph-mon[101180]: osdmap e503: 8 total, 8 up, 7 in 2024-09-14T04:29:41.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:41.719 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:41.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:41.873 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:41.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:41.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:42.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:42.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:42.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:42.248 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:42.291 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:42.393 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:42.434 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:42.443 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 11 16 22663 22647 128.638 124.375 0.0051786 0.00776668 2024-09-14T04:29:42.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:42.570 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:42.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:42.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:42 smithi012 ceph-mon[101180]: pgmap v1587: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 98 active+clean; 1.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 6917/52293 objects degraded (13.227%); 176 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T04:29:42.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:42.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:42 smithi001 ceph-mon[112165]: pgmap v1587: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 98 active+clean; 1.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 6917/52293 objects degraded (13.227%); 176 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T04:29:42.692 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:42 smithi001 ceph-mon[107399]: pgmap v1587: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 98 active+clean; 1.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 6917/52293 objects degraded (13.227%); 176 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T04:29:42.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:42.819 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:42.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:42.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:42.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:43.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.008 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:43.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:43.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:43.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.286 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:43.387 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:43.428 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:43.443 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 12 16 24361 24345 126.761 106.125 0.00382671 0.00788017 2024-09-14T04:29:43.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:43 smithi001 ceph-mon[112165]: Health check failed: Degraded data redundancy: 6917/52293 objects degraded (13.227%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:43.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:43 smithi001 ceph-mon[107399]: Health check failed: Degraded data redundancy: 6917/52293 objects degraded (13.227%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:43.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.558 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:43.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:43.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:43 smithi012 ceph-mon[101180]: Health check failed: Degraded data redundancy: 6917/52293 objects degraded (13.227%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:43.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:43.808 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:43.839 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:43.941 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:43.977 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:43.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:44.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:44.111 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:44.213 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.238 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:44.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:44.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:44.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:44.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:29:44.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 13 16 25886 25870 124.339 95.3125 0.00906535 0.00803586 2024-09-14T04:29:44.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:44.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:44.565 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:44 smithi001 ceph-mon[112165]: pgmap v1588: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 97 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 6917/54465 objects degraded (12.700%); 766/54465 objects misplaced (1.406%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2024-09-14T04:29:44.565 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:44 smithi001 ceph-mon[107399]: pgmap v1588: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 97 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 6917/54465 objects degraded (12.700%); 766/54465 objects misplaced (1.406%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2024-09-14T04:29:44.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:44 smithi012 ceph-mon[101180]: pgmap v1588: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 97 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 6917/54465 objects degraded (12.700%); 766/54465 objects misplaced (1.406%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2024-09-14T04:29:44.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:44.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:44.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:44.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.788 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:44.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:44.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:44.919 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:44.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:45.021 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:45.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:45.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:45.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:45.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:45.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:45.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 14 16 27676 27660 123.448 111.875 0.00448657 0.00809179 2024-09-14T04:29:45.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:45.511 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:45.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:45.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:45.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:45.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:45.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:45.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:45.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:46.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.055 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:46.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:46.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:46.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:46.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:46.324 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:46.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:46.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 15 16 29617 29601 123.305 121.312 0.00807016 0.00810421 2024-09-14T04:29:46.462 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:46.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:46.599 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:46.701 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:46.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:46.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:46.878 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:46.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:46 smithi001 ceph-mon[107399]: pgmap v1589: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 138 MiB/s wr, 2.20k op/s; 6917/77925 objects degraded (8.876%); 766/77925 objects misplaced (0.983%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2024-09-14T04:29:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:46] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-09-14T04:29:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:46 smithi001 ceph-mon[112165]: pgmap v1589: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 138 MiB/s wr, 2.20k op/s; 6917/77925 objects degraded (8.876%); 766/77925 objects misplaced (0.983%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2024-09-14T04:29:46.980 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:47.037 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:46 smithi012 ceph-mon[101180]: pgmap v1589: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 138 MiB/s wr, 2.20k op/s; 6917/77925 objects degraded (8.876%); 766/77925 objects misplaced (0.983%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2024-09-14T04:29:47.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:47.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:47.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:47.300 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:47.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:47.444 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:47.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 16 16 31389 31373 122.52 110.75 0.0150927 0.00815615 2024-09-14T04:29:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:47.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:47.574 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:47.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:47.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:47.716 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:47.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:47.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:47.954 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:47.991 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:47.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:48.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:48.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:48.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[107399]: pgmap v1590: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 6917/81384 objects degraded (8.499%); 194/81384 objects misplaced (0.238%); 6.4 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:47 smithi001 ceph-mon[112165]: pgmap v1590: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 6917/81384 objects degraded (8.499%); 194/81384 objects misplaced (0.238%); 6.4 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-14T04:29:48.234 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:48.272 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:48.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:47 smithi012 ceph-mon[101180]: pgmap v1590: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 6917/81384 objects degraded (8.499%); 194/81384 objects misplaced (0.238%); 6.4 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-14T04:29:48.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:48.404 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:48.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 17 16 33182 33166 121.904 112.062 0.00527243 0.00819723 2024-09-14T04:29:48.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:48.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:48.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:48.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:48.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:48.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:48.831 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:48.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:48.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:48.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:49.012 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 3, 7, 6] out_osds: [2] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:29:49.012 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:29:49.012 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 4, 5, 3, 7, 6] 2024-09-14T04:29:49.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd out 1 2024-09-14T04:29:49.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:49.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:49.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:49.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:48 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 6917/81384 objects degraded (8.499%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:49.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:48 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 6917/81384 objects degraded (8.499%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:49.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:49.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:49.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:49.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:48 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 6917/81384 objects degraded (8.499%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.349 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:49.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:49.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 18 16 34279 34263 118.941 68.5625 0.00262279 0.00820675 2024-09-14T04:29:49.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:49.472 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:49.574 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:49.596 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:49.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:49.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:49.823 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:49.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:49.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:49.978 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:49.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:50.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:50.116 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:50.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:50.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:49 smithi001 ceph-mon[112165]: osdmap e504: 8 total, 8 up, 7 in 2024-09-14T04:29:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:49 smithi001 ceph-mon[112165]: pgmap v1592: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 6917/82884 objects degraded (8.345%); 194/82884 objects misplaced (0.234%); 5.7 MiB/s, 90 objects/s recovering 2024-09-14T04:29:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:49 smithi001 ceph-mon[107399]: osdmap e504: 8 total, 8 up, 7 in 2024-09-14T04:29:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:49 smithi001 ceph-mon[107399]: pgmap v1592: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 6917/82884 objects degraded (8.345%); 194/82884 objects misplaced (0.234%); 5.7 MiB/s, 90 objects/s recovering 2024-09-14T04:29:50.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:49 smithi012 ceph-mon[101180]: osdmap e504: 8 total, 8 up, 7 in 2024-09-14T04:29:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:49 smithi012 ceph-mon[101180]: pgmap v1592: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 6917/82884 objects degraded (8.345%); 194/82884 objects misplaced (0.234%); 5.7 MiB/s, 90 objects/s recovering 2024-09-14T04:29:50.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:50.392 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:50.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 19 16 35116 35100 115.434 52.3125 0.00465508 0.00865653 2024-09-14T04:29:50.494 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:50.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:50.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:50.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:50.650 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:50.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:50.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:50.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:50.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:50.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:50.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:51.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:51.052 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:51.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:51.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[112165]: osdmap e505: 8 total, 8 up, 7 in 2024-09-14T04:29:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:51.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:51.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[107399]: osdmap e505: 8 total, 8 up, 7 in 2024-09-14T04:29:51.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:51.222 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:51.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:51.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:51.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:50 smithi012 ceph-mon[101180]: osdmap e505: 8 total, 8 up, 7 in 2024-09-14T04:29:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:51.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:51.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:29:51.444541+0000 min lat: 0.00130141 max lat: 2.15405 avg lat: 0.00871515 2024-09-14T04:29:51.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:29:51.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 20 16 36717 36701 114.666 100.062 0.00691753 0.00871515 2024-09-14T04:29:51.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:51.551 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:51.595 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:51.697 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:51.738 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:51.840 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:51.892 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:51.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:51.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:52.022 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:52.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:52.086 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:29:52.118 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:51 smithi001 ceph-mon[107399]: pgmap v1594: 109 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 5964/101913 objects degraded (5.852%); 881/101913 objects misplaced (0.864%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:29:52.119 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:51 smithi001 ceph-mon[112165]: pgmap v1594: 109 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 5964/101913 objects degraded (5.852%); 881/101913 objects misplaced (0.864%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:29:52.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:52.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:52.249 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:52.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:52.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:51 smithi012 ceph-mon[101180]: pgmap v1594: 109 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 5964/101913 objects degraded (5.852%); 881/101913 objects misplaced (0.864%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:29:52.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:52.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:52.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:52.444 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 21 16 38154 38138 113.482 89.8125 0.0122565 0.00880512 2024-09-14T04:29:52.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:52.576 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:52.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:52.716 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:52.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:52.818 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:52.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:52.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:52.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:53.002 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:53.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:53.090 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:53.091 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:53.091 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:53.091 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:53.091 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:53.091 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:53.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:53.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:53.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:53.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:53.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:29:53.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:53.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:53.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 22 16 39861 39845 113.173 106.688 0.00462102 0.00882909 2024-09-14T04:29:53.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.570 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:53.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:53.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.715 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:53.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:53.816 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:53.858 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:53.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:53.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:53.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:54.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:54.117 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:29:54.117 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.46352.0:40107 14.1 e505/504 ondisk, result = 0)", 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.647585+0000", 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0053628149999999999, 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053868340000000001, 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647585+0000", 2024-09-14T04:29:54.118 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647585+0000", 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647585+0000", 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7100000000000001e-07 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.119 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647590+0000", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0250000000000002e-06 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647601+0000", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0828e-05 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647608+0000", 2024-09-14T04:29:54.120 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6800000000000004e-06 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648537+0000", 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000929377 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648545+0000", 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0420000000000003e-06 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.121 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:40115 14.1 14.b6c05031 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.647977+0000", 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0049708620000000004, 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005051201, 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:29:54.122 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 40115 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647977+0000", 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647977+0000", 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.123 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.647978+0000", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.55e-06 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648002+0000", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4307999999999998e-05 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648016+0000", 2024-09-14T04:29:54.124 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3355e-05 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648024+0000", 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9959999999999993e-06 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.46352.0:40108 14.9 e505/494)", 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.648325+0000", 2024-09-14T04:29:54.125 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0046227799999999999, 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047485499999999998, 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648325+0000", 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.126 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648325+0000", 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648325+0000", 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8899999999999999e-07 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648329+0000", 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0419999999999996e-06 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.127 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648351+0000", 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1509e-05 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648358+0000", 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.128 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop_reply(client.46352.0:40108 14.9 e505/494)", 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.648790+0000", 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0041571760000000003, 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004318629, 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648790+0000", 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.129 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648790+0000", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648791+0000", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9299999999999997e-07 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648795+0000", 2024-09-14T04:29:54.130 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3109999999999999e-06 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648806+0000", 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0814e-05 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648812+0000", 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1129999999999997e-06 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.131 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.46352.0:40117 14.8 e505/500 14:16ca3b2a:::benchmark_data_smithi001_337422_object40116:head v 505'2524, mlcod=0'0)", 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.650392+0000", 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0025554810000000001, 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027560879999999999, 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650392+0000", 2024-09-14T04:29:54.132 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650392+0000", 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650392+0000", 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.89e-07 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.133 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650443+0000", 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0260999999999999e-05 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650450+0000", 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8169999999999999e-06 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650456+0000", 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1499999999999998e-06 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.134 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650558+0000", 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102144 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650584+0000", 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6098999999999999e-05 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.135 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_repop(client.46352.0:40116 14.2 e505/494 14:4e0276ce:::benchmark_data_smithi001_337422_object40115:head v 505'2545, mlcod=505'2543)", 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.650481+0000", 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.0024662909999999998, 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027140010000000002, 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "started", 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650481+0000", 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.136 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650481+0000", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650482+0000", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9000000000000002e-07 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650512+0000", 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0725000000000001e-05 2024-09-14T04:29:54.137 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650519+0000", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.99e-06 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.650524+0000", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5600000000000004e-06 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651065+0000", 2024-09-14T04:29:54.138 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054168399999999996 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651084+0000", 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8853000000000001e-05 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:40120 14.1 14.64dc961 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.651754+0000", 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.001193057, 2024-09-14T04:29:54.139 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0014854829999999999, 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "queued for pg", 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 40120 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651754+0000", 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.140 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651754+0000", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651756+0000", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.544e-06 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651779+0000", 2024-09-14T04:29:54.141 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3283000000000001e-05 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651792+0000", 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3113e-05 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.142 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.651800+0000", 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.174e-06 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:40108 14.9 14:9682447e:::benchmark_data_smithi001_337422_object40107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.639157+0000", 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.013790801, 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.014134857000000001, 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.143 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 40108 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639157+0000", 2024-09-14T04:29:54.144 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639157+0000", 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639157+0000", 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9299999999999998e-07 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.145 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639178+0000", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1279000000000001e-05 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639186+0000", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.435e-06 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.639191+0000", 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2020000000000003e-06 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.146 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.642534+0000", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003342504 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.642664+0000", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013052300000000001 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.644259+0000", 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001595251 2024-09-14T04:29:54.147 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.653139+0000", 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0088794110000000002 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.653175+0000", 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6529000000000002e-05 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.148 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:40110 14.9 14:9f12d588:::benchmark_data_smithi001_337422_object40109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:53.640610+0000", 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 0.012337280000000001, 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012759408, 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 40110 2024-09-14T04:29:54.149 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640610+0000", 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640610+0000", 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:29:54.150 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640611+0000", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2160000000000001e-06 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640636+0000", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4915000000000001e-05 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640646+0000", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5249999999999994e-06 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:29:54.151 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.640653+0000", 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1180000000000003e-06 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648708+0000", 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080547350000000004 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.648863+0000", 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015521600000000001 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.152 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.649010+0000", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014693200000000001 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.653211+0000", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.004201481 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:53.653248+0000", 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6400999999999999e-05 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.153 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:29:54.154 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.154 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:29:54.154 INFO:teuthology.orchestra.run.smithi001.stdout: ], 2024-09-14T04:29:54.154 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 9 2024-09-14T04:29:54.154 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:29:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:54.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:54.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:54.289 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:53 smithi012 ceph-mon[101180]: pgmap v1595: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 5964/104490 objects degraded (5.708%); 687/104490 objects misplaced (0.657%); 8.1 MiB/s, 128 objects/s recovering 2024-09-14T04:29:54.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:54.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:54.445 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:53 smithi001 ceph-mon[107399]: pgmap v1595: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 5964/104490 objects degraded (5.708%); 687/104490 objects misplaced (0.657%); 8.1 MiB/s, 128 objects/s recovering 2024-09-14T04:29:54.445 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:53 smithi001 ceph-mon[112165]: pgmap v1595: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 5964/104490 objects degraded (5.708%); 687/104490 objects misplaced (0.657%); 8.1 MiB/s, 128 objects/s recovering 2024-09-14T04:29:54.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 23 16 41355 41339 112.312 93.375 0.0229102 0.00889787 2024-09-14T04:29:54.521 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.562 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:54.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:54.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:54.704 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:54.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:54.844 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:54.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:54.946 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:54.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:54.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:54.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:54.997 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:29:55.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:55.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5964/104490 objects degraded (5.708%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4178209762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5964/104490 objects degraded (5.708%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4178209762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:54 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:55.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5964/104490 objects degraded (5.708%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:55.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4178209762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.265 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:54 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:29:55.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:55.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:29:55.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:55.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:55.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 24 16 42987 42971 111.882 102 0.00577822 0.00893189 2024-09-14T04:29:55.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:55.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=cleanup t=2024-09-14T04:29:55.262941971Z level=info msg="Completed cleanup jobs" duration=12.314166ms 2024-09-14T04:29:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=plugins.update.checker t=2024-09-14T04:29:55.424247646Z level=info msg="Update check succeeded" duration=64.887262ms 2024-09-14T04:29:55.596 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana.update.checker t=2024-09-14T04:29:55.424507403Z level=info msg="Update check succeeded" duration=59.302681ms 2024-09-14T04:29:55.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.694 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:55.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:55.795 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:55.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:55.942 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:55.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:55.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:56.084 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:56.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:56.212 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:56.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:56.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:56.336 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:56.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-mon[101180]: osdmap e506: 8 total, 8 up, 7 in 2024-09-14T04:29:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:55 smithi012 ceph-mon[101180]: pgmap v1597: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s; 4941/124455 objects degraded (3.970%); 1077/124455 objects misplaced (0.865%); 10 MiB/s, 164 objects/s recovering 2024-09-14T04:29:56.358 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:56.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:56.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 25 16 44023 44007 109.996 64.75 0.00224198 0.00894288 2024-09-14T04:29:56.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:56.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:56.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:56.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[107399]: osdmap e506: 8 total, 8 up, 7 in 2024-09-14T04:29:56.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[107399]: pgmap v1597: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s; 4941/124455 objects degraded (3.970%); 1077/124455 objects misplaced (0.865%); 10 MiB/s, 164 objects/s recovering 2024-09-14T04:29:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:29:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:29:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[112165]: osdmap e506: 8 total, 8 up, 7 in 2024-09-14T04:29:56.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:55 smithi001 ceph-mon[112165]: pgmap v1597: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s; 4941/124455 objects degraded (3.970%); 1077/124455 objects misplaced (0.865%); 10 MiB/s, 164 objects/s recovering 2024-09-14T04:29:56.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:56.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:56.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:56.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:56.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:56.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:56.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:56.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:29:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:29:56] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:29:56.980 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:56.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:57.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:57.112 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:57.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:57.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:57.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:56 smithi012 ceph-mon[101180]: osdmap e507: 8 total, 8 up, 7 in 2024-09-14T04:29:57.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:57.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:57.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 26 16 44756 44740 107.528 45.8125 0.0119932 0.00929472 2024-09-14T04:29:57.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:56 smithi001 ceph-mon[107399]: osdmap e507: 8 total, 8 up, 7 in 2024-09-14T04:29:57.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:56 smithi001 ceph-mon[112165]: osdmap e507: 8 total, 8 up, 7 in 2024-09-14T04:29:57.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:57.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:57.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:57.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:57.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:57.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:57.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:57.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:57.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:58.049 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:29:58.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:29:58.192 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:29:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.218 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:58.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:29:58.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:58 smithi001 ceph-mon[107399]: osdmap e508: 8 total, 8 up, 7 in 2024-09-14T04:29:58.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:58 smithi001 ceph-mon[107399]: pgmap v1600: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 4941/126945 objects degraded (3.892%); 1077/126945 objects misplaced (0.848%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:29:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:58 smithi001 ceph-mon[112165]: osdmap e508: 8 total, 8 up, 7 in 2024-09-14T04:29:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:58 smithi001 ceph-mon[112165]: pgmap v1600: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 4941/126945 objects degraded (3.892%); 1077/126945 objects misplaced (0.848%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:29:58.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:58.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:58 smithi012 ceph-mon[101180]: osdmap e508: 8 total, 8 up, 7 in 2024-09-14T04:29:58.348 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:58 smithi012 ceph-mon[101180]: pgmap v1600: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 4941/126945 objects degraded (3.892%); 1077/126945 objects misplaced (0.848%); 8.6 MiB/s, 137 objects/s recovering 2024-09-14T04:29:58.356 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:58.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 27 16 46310 46294 107.142 97.125 0.0119234 0.00932815 2024-09-14T04:29:58.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:29:58.490 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:29:58.592 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:58.636 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:58.738 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:29:58.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:29:58.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:58.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:58.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:58.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:59.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:59.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:59.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:59.169 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:29:59.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:59.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:59.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:59.440 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:59.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 28 16 47792 47776 106.623 92.625 0.0069305 0.00937332 2024-09-14T04:29:59.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:29:59.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:29:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:29:59.593 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:29:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:59.595 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:29:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:29:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:29:59.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:29:59.721 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:29:59.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:29:59.856 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:29:59.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:29:59.989 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:29:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:29:59.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:00.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:00.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:00.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:00.237 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:00.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:00.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:00.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:00.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 29 16 49306 49290 106.209 94.625 0.00699642 0.00940974 2024-09-14T04:30:00.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pgmap v1601: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 4941/128193 objects degraded (3.854%); 1077/128193 objects misplaced (0.840%); 6.6 MiB/s, 105 objects/s recovering 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: [WRN] PG_DEGRADED: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pg 14.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[112165]: pg 14.d is active+recovery_wait+undersized+degraded+remapped, acting [4,5] 2024-09-14T04:30:00.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pgmap v1601: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 4941/128193 objects degraded (3.854%); 1077/128193 objects misplaced (0.840%); 6.6 MiB/s, 105 objects/s recovering 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: [WRN] PG_DEGRADED: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pg 14.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-14T04:30:00.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:00 smithi001 ceph-mon[107399]: pg 14.d is active+recovery_wait+undersized+degraded+remapped, acting [4,5] 2024-09-14T04:30:00.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:00.559 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pgmap v1601: 109 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 4941/128193 objects degraded (3.854%); 1077/128193 objects misplaced (0.840%); 6.6 MiB/s, 105 objects/s recovering 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: [WRN] PG_DEGRADED: Degraded data redundancy: 4941/126945 objects degraded (3.892%), 5 pgs degraded 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-14T04:30:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-14T04:30:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pg 14.c is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-14T04:30:00.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:00 smithi012 ceph-mon[101180]: pg 14.d is active+recovery_wait+undersized+degraded+remapped, acting [4,5] 2024-09-14T04:30:00.662 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:00.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:00.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:00.834 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:00.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:00.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:00.973 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:00.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:01.074 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:01.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:01.211 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:01.221 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:01.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:01.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:01.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:01.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 30 16 50888 50872 105.965 98.875 0.0112675 0.00943116 2024-09-14T04:30:01.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:01.597 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:01.627 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:30:01.628 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:30:01.665 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:30:01.665 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:30:01.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:01.711 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:30:01.711 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:30:01.736 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:30:01.736 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:30:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.748 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:01.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:01.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:30:01.771 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:30:01.772 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:30:01.809 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/844043934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:01.809 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/844043934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:01.812 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:30:01.812 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:30:01.845 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:30:01.845 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:30:01.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:01.881 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:30:01.882 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:30:01.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:01.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:02.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:02.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/844043934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:02.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:02.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:02.293 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.336 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:02.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:02.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:02.445 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 31 16 52188 52172 105.166 81.25 0.00665461 0.00950311 2024-09-14T04:30:02.478 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:02.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.621 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:02.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:02.722 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:02.762 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:02.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:02.895 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:02.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:02 smithi001 ceph-mon[107399]: pgmap v1602: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 4941/143787 objects degraded (3.436%); 304/143787 objects misplaced (0.211%); 7.4 MiB/s, 118 objects/s recovering 2024-09-14T04:30:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:02 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:30:02.803+0000 7fbfffcba640 -1 mon.a@0(leader).osd e508 definitely_dead 0 2024-09-14T04:30:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:02 smithi001 ceph-mon[112165]: pgmap v1602: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 4941/143787 objects degraded (3.436%); 304/143787 objects misplaced (0.211%); 7.4 MiB/s, 118 objects/s recovering 2024-09-14T04:30:02.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:03.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:02 smithi012 ceph-mon[101180]: pgmap v1602: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 4941/143787 objects degraded (3.436%); 304/143787 objects misplaced (0.211%); 7.4 MiB/s, 118 objects/s recovering 2024-09-14T04:30:03.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:03.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:03.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:03.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:03.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:03.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 32 16 53642 53626 104.719 90.875 0.0135726 0.00954369 2024-09-14T04:30:03.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:03.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:03.606 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:03.665 INFO:teuthology.orchestra.run.smithi001.stderr:marked out osd.1. 2024-09-14T04:30:03.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.733 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:03.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:03.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:03.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:03.962 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:03.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3394778573' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:03.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:03.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[107399]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-14T04:30:03.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3394778573' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:03.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:03.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:03 smithi001 ceph-mon[112165]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-14T04:30:03.985 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:03.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:04.087 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:04.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3394778573' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:03 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-14T04:30:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:03 smithi012 ceph-mon[101180]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-14T04:30:04.110 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:04.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:04.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:04.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:04.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:04.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 33 16 53971 53955 102.169 20.5625 0.0027716 0.00955446 2024-09-14T04:30:04.468 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:04.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:04.592 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:04.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:04.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:04.742 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:04.828 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:04.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:04.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:04 smithi012 ceph-mon[101180]: pgmap v1603: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s; 4941/146508 objects degraded (3.373%); 304/146508 objects misplaced (0.207%); 6.0 MiB/s, 96 objects/s recovering 2024-09-14T04:30:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:04 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-14T04:30:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:04 smithi012 ceph-mon[101180]: osdmap e509: 8 total, 8 up, 6 in 2024-09-14T04:30:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4941/146508 objects degraded (3.373%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:04.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:04 smithi012 ceph-mon[101180]: osdmap e510: 8 total, 8 up, 6 in 2024-09-14T04:30:04.867 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:04.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[107399]: pgmap v1603: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s; 4941/146508 objects degraded (3.373%); 304/146508 objects misplaced (0.207%); 6.0 MiB/s, 96 objects/s recovering 2024-09-14T04:30:04.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-14T04:30:04.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[107399]: osdmap e509: 8 total, 8 up, 6 in 2024-09-14T04:30:04.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4941/146508 objects degraded (3.373%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:04.944 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[107399]: osdmap e510: 8 total, 8 up, 6 in 2024-09-14T04:30:04.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[112165]: pgmap v1603: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s; 4941/146508 objects degraded (3.373%); 304/146508 objects misplaced (0.207%); 6.0 MiB/s, 96 objects/s recovering 2024-09-14T04:30:04.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-14T04:30:04.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[112165]: osdmap e509: 8 total, 8 up, 6 in 2024-09-14T04:30:04.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4941/146508 objects degraded (3.373%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:04.945 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:04 smithi001 ceph-mon[112165]: osdmap e510: 8 total, 8 up, 6 in 2024-09-14T04:30:04.969 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:04.992 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:04.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:05.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:05.117 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:05.219 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:05.242 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:05.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:05.344 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:05.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:05.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 34 16 53971 53955 99.1643 0 - 0.00955446 2024-09-14T04:30:05.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:05.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:05.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:05.612 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:05.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:30:05.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:05.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:05.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:05.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:05.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:05.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:05.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:05 smithi001 ceph-mon[112165]: osdmap e511: 8 total, 8 up, 6 in 2024-09-14T04:30:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:05.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:05 smithi001 ceph-mon[107399]: osdmap e511: 8 total, 8 up, 6 in 2024-09-14T04:30:06.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:06.062 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:05 smithi012 ceph-mon[101180]: osdmap e511: 8 total, 8 up, 6 in 2024-09-14T04:30:06.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:06.196 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:06.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:06.323 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:06.425 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:06.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 35 16 54805 54789 97.8203 26.0625 0.0248487 0.010213 2024-09-14T04:30:06.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:06.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:06.594 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:06.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:06.737 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:06.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:06.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:06.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:06.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[107399]: pgmap v1607: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 KiB/s rd, 109 MiB/s wr, 1.75k op/s; 5135/159633 objects degraded (3.217%); 304/159633 objects misplaced (0.190%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:30:06.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:30:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[107399]: osdmap e512: 8 total, 8 up, 6 in 2024-09-14T04:30:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[112165]: pgmap v1607: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 KiB/s rd, 109 MiB/s wr, 1.75k op/s; 5135/159633 objects degraded (3.217%); 304/159633 objects misplaced (0.190%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:30:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:30:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-mon[112165]: osdmap e512: 8 total, 8 up, 6 in 2024-09-14T04:30:06.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:06] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-14T04:30:06.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:07.007 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:06 smithi012 ceph-mon[101180]: pgmap v1607: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 KiB/s rd, 109 MiB/s wr, 1.75k op/s; 5135/159633 objects degraded (3.217%); 304/159633 objects misplaced (0.190%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:30:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:06 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:30:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:06 smithi012 ceph-mon[101180]: osdmap e512: 8 total, 8 up, 6 in 2024-09-14T04:30:07.108 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:07.141 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:07.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:07.277 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:07.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:07.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:07.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 36 16 55803 55787 96.8356 62.375 0.0277816 0.01032 2024-09-14T04:30:07.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:07.561 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:07.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:07.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:07.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:07.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:07.850 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:07.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:07.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:07.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:08.091 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:08.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:08.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:08.268 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:08.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:08.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:08 smithi001 ceph-mon[112165]: osdmap e513: 8 total, 8 up, 6 in 2024-09-14T04:30:08.389 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:08 smithi001 ceph-mon[112165]: pgmap v1610: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.4 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 5135/161139 objects degraded (3.187%); 3699/161139 objects misplaced (2.296%); 19 MiB/s, 311 objects/s recovering 2024-09-14T04:30:08.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:08 smithi001 ceph-mon[107399]: osdmap e513: 8 total, 8 up, 6 in 2024-09-14T04:30:08.389 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:08 smithi001 ceph-mon[107399]: pgmap v1610: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.4 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 5135/161139 objects degraded (3.187%); 3699/161139 objects misplaced (2.296%); 19 MiB/s, 311 objects/s recovering 2024-09-14T04:30:08.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:08.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:08.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 37 16 56779 56763 95.867 61 0.00489655 0.010425 2024-09-14T04:30:08.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:08.548 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:08 smithi012 ceph-mon[101180]: osdmap e513: 8 total, 8 up, 6 in 2024-09-14T04:30:08.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:08 smithi012 ceph-mon[101180]: pgmap v1610: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.4 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 5135/161139 objects degraded (3.187%); 3699/161139 objects misplaced (2.296%); 19 MiB/s, 311 objects/s recovering 2024-09-14T04:30:08.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.692 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:08.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:08.723 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:30:08.723 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:30:08.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:08.831 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:08.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:08.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:08.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:09.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:09.114 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:09.217 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:09.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:09.271 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:30:09.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:09.396 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 7, 6] out_osds: [2, 1] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:30:09.396 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:30:09.396 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-14T04:30:09.397 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:30:09.397 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:30:09.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-14T04:30:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:09.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:09.435 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:30:09.435 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5135/161139 objects degraded (3.187%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:09.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:09.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:30:09.436 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[112165]: osdmap e514: 8 total, 8 up, 6 in 2024-09-14T04:30:09.438 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:30:09.439 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5135/161139 objects degraded (3.187%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:09.439 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:09.439 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:30:09.439 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:09 smithi001 ceph-mon[107399]: osdmap e514: 8 total, 8 up, 6 in 2024-09-14T04:30:09.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 38 16 57785 57769 94.9987 62.875 0.00687797 0.0105113 2024-09-14T04:30:09.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:09.537 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:09.594 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:30:09.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:30:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5135/161139 objects degraded (3.187%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:09 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2868764636' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:30:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:09 smithi012 ceph-mon[101180]: osdmap e514: 8 total, 8 up, 6 in 2024-09-14T04:30:09.639 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:09.662 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:09.764 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:09.787 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:09.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:09.911 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:09.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:10.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:10.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:10.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:10.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:10.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:10.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:10.286 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:10.359 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:10.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:10.414 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:10.421 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:10.446 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 39 15 58041 58026 92.9747 16.0625 0.00893035 0.0107499 2024-09-14T04:30:10.523 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.569 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:10 smithi012 ceph-mon[101180]: pgmap v1612: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.8 KiB/s rd, 29 MiB/s wr, 475 op/s; 5135/163041 objects degraded (3.150%); 7053/163041 objects misplaced (4.326%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:30:10.649 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:10 smithi001 ceph-mon[107399]: pgmap v1612: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.8 KiB/s rd, 29 MiB/s wr, 475 op/s; 5135/163041 objects degraded (3.150%); 7053/163041 objects misplaced (4.326%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:30:10.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:10 smithi001 ceph-mon[112165]: pgmap v1612: 109 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovery_wait+degraded, 10 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.8 KiB/s rd, 29 MiB/s wr, 475 op/s; 5135/163041 objects degraded (3.150%); 7053/163041 objects misplaced (4.326%); 15 MiB/s, 238 objects/s recovering 2024-09-14T04:30:10.669 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:10.708 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:10.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:10.846 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:10.948 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:10.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:10.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:11.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:11.127 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:11.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:11.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:11.269 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:11.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:11.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:11.408 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:11.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:30:11.446831+0000 min lat: 0.00130141 max lat: 2.15405 avg lat: 0.0108378 2024-09-14T04:30:11.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:30:11.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 40 16 59036 59020 92.2034 62.125 0.00648943 0.0108378 2024-09-14T04:30:11.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:11.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:11 smithi012 ceph-mon[101180]: osdmap e515: 8 total, 8 up, 6 in 2024-09-14T04:30:11.647 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:11.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:11.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:11 smithi001 ceph-mon[107399]: osdmap e515: 8 total, 8 up, 6 in 2024-09-14T04:30:11.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:11 smithi001 ceph-mon[112165]: osdmap e515: 8 total, 8 up, 6 in 2024-09-14T04:30:11.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:11.813 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:11.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:11.954 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:11.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:11.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:12.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:12.096 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:12.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:12.212 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:12.233 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:12.318 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:30:12.335 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:12.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:12.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 41 16 60010 59994 91.4392 60.875 0.00964071 0.0109307 2024-09-14T04:30:12.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.546 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:12.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:12.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:12 smithi012 ceph-mon[101180]: pgmap v1614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s; 27456/172674 objects degraded (15.900%); 6749/172674 objects misplaced (3.909%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:30:12.646 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:12 smithi001 ceph-mon[107399]: pgmap v1614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s; 27456/172674 objects degraded (15.900%); 6749/172674 objects misplaced (3.909%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:30:12.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:12 smithi001 ceph-mon[112165]: pgmap v1614: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s; 27456/172674 objects degraded (15.900%); 6749/172674 objects misplaced (3.909%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:30:12.648 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:12.679 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:12.781 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:12.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:12.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:12.944 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:12.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:13.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:13.085 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:13.186 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.223 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:13.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:13.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:13.361 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:13.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 42 16 60885 60869 90.5641 54.6875 0.00471722 0.0110351 2024-09-14T04:30:13.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:13.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:13 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:30:13.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:13 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:30:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:13.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:13 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2024-09-14T04:30:13.609 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:13.646 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:13.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.785 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:13.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:13.923 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:13.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:14.025 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:14.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:14.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:14.203 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:14.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:14.342 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:14.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:14.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 43 16 61794 61778 89.7791 56.8125 0.0115106 0.0111318 2024-09-14T04:30:14.481 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:14.583 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:14.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:14 smithi012 ceph-mon[101180]: pgmap v1615: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s; 27456/174255 objects degraded (15.756%); 6459/174255 objects misplaced (3.707%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:30:14.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/172674 objects degraded (15.900%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:14.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:14.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:14 smithi001 ceph-mon[107399]: pgmap v1615: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s; 27456/174255 objects degraded (15.756%); 6459/174255 objects misplaced (3.707%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:30:14.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/172674 objects degraded (15.900%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:14 smithi001 ceph-mon[112165]: pgmap v1615: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s; 27456/174255 objects degraded (15.756%); 6459/174255 objects misplaced (3.707%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:30:14.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/172674 objects degraded (15.900%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:14.725 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:14.760 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:14.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:14.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:15.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:15.046 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:15.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:15.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:15.332 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:15.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:15.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 44 16 62715 62699 89.0468 57.5625 0.0427036 0.0112231 2024-09-14T04:30:15.482 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:15.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:15.617 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:15.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:15.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:15.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:15.908 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:15.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:16.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:16.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:16.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:16.173 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:16.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:16.313 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:16.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:16.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 45 16 63707 63691 88.4457 62 0.0168854 0.0113007 2024-09-14T04:30:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:16.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:16.557 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:16.588 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:16.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:16.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:16.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:16.880 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:16.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:16.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:16 smithi001 ceph-mon[107399]: pgmap v1616: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 27456/185664 objects degraded (14.788%); 6459/185664 objects misplaced (3.479%); 4.7 MiB/s, 74 objects/s recovering 2024-09-14T04:30:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:16 smithi001 ceph-mon[112165]: pgmap v1616: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 27456/185664 objects degraded (14.788%); 6459/185664 objects misplaced (3.479%); 4.7 MiB/s, 74 objects/s recovering 2024-09-14T04:30:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:16] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-14T04:30:16.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:17.019 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:16 smithi012 ceph-mon[101180]: pgmap v1616: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 27456/185664 objects degraded (14.788%); 6459/185664 objects misplaced (3.479%); 4.7 MiB/s, 74 objects/s recovering 2024-09-14T04:30:17.121 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:17.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:17.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:17.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:17.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:17.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:17.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:17.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:17.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 46 16 64593 64577 87.7267 55.375 0.00465877 0.0113931 2024-09-14T04:30:17.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:17.564 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:17.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:17.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:17.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:17.802 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.837 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:17.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:17.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:17.981 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:17.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:18.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:18.119 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:18.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:18.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:18.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:18.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:18.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 47 16 65525 65509 87.0995 58.25 0.0202447 0.0114748 2024-09-14T04:30:18.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:18.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:18.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:18.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:18.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.816 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:18.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:18.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:18.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:18.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:18.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:18 smithi001 ceph-mon[112165]: pgmap v1617: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 27456/187401 objects degraded (14.651%); 6245/187401 objects misplaced (3.332%); 6.1 MiB/s, 97 objects/s recovering 2024-09-14T04:30:18.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:18 smithi001 ceph-mon[107399]: pgmap v1617: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 27456/187401 objects degraded (14.651%); 6245/187401 objects misplaced (3.332%); 6.1 MiB/s, 97 objects/s recovering 2024-09-14T04:30:19.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:18 smithi012 ceph-mon[101180]: pgmap v1617: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 27456/187401 objects degraded (14.651%); 6245/187401 objects misplaced (3.332%); 6.1 MiB/s, 97 objects/s recovering 2024-09-14T04:30:19.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:19.207 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:19.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:19.339 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:19.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:19.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 48 16 66464 66448 86.5075 58.6875 0.0101119 0.0115536 2024-09-14T04:30:19.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:19.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.511 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:19.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:19.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:19.652 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:19.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.797 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:19.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:19.898 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:19.928 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:19.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:19.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/187401 objects degraded (14.651%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:19.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/187401 objects degraded (14.651%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:20.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:20.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:20.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/187401 objects degraded (14.651%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:20.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:20.216 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:20.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.366 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:20.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 49 16 67378 67362 85.9078 57.125 0.0206627 0.0116345 2024-09-14T04:30:20.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:20.503 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:20.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:20.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:20.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:20.784 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:20.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:20.889 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:20 smithi001 ceph-mon[112165]: pgmap v1618: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/188262 objects degraded (14.584%); 6245/188262 objects misplaced (3.317%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:30:20.889 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:20.889 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:20 smithi001 ceph-mon[107399]: pgmap v1618: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/188262 objects degraded (14.584%); 6245/188262 objects misplaced (3.317%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:30:20.889 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:20.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:20.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:21.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:21.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:21.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:21.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:20 smithi012 ceph-mon[101180]: pgmap v1618: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/188262 objects degraded (14.584%); 6245/188262 objects misplaced (3.317%); 5.1 MiB/s, 80 objects/s recovering 2024-09-14T04:30:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:21.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:21.211 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:21.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:21.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:21.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 50 16 68317 68301 85.3634 58.6875 0.01174 0.0117087 2024-09-14T04:30:21.462 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:21.497 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:21.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:21.632 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:21.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:21.733 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:21.768 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:21.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:21.910 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:21.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:22.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:22.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:22.135 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:22.172 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:22.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:22.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:22.426 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:22.447 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 51 16 69200 69184 84.7716 55.1875 0.00956468 0.0117899 2024-09-14T04:30:22.465 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:22.567 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:22.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:22.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:22.743 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:22.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:22.888 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:22.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:22.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:22.904 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:30:22.904 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:30:22.905 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:22 smithi001 ceph-mon[107399]: pgmap v1619: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 27456/199749 objects degraded (13.745%); 6245/199749 objects misplaced (3.126%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:30:22.905 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:22 smithi001 ceph-mon[112165]: pgmap v1619: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 27456/199749 objects degraded (13.745%); 6245/199749 objects misplaced (3.126%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:30:22.917 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:22.936 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:30:22.937 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:30:22.970 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:30:22.971 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:30:22.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:22.999 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:30:22.999 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:30:23.029 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:30:23.030 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:30:23.035 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:23.058 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:30:23.059 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:30:23.086 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:30:23.086 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:30:23.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:22 smithi012 ceph-mon[101180]: pgmap v1619: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 27456/199749 objects degraded (13.745%); 6245/199749 objects misplaced (3.126%); 4.5 MiB/s, 71 objects/s recovering 2024-09-14T04:30:23.113 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:30:23.113 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:30:23.137 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:23.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:23.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:23.305 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:23.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:23.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:23.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 52 16 70135 70119 84.2651 58.4375 0.0191734 0.011861 2024-09-14T04:30:23.545 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:23.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:23.682 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:23.716 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:23.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:23.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:23.956 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:23.963 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:23 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1154655753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:23.963 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:23 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1154655753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:23.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:23.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:24.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:24.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:23 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1154655753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:24.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:24.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:24.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:24.358 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:30:24.358 INFO:teuthology.orchestra.run.smithi001.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:30:24.358 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:30:24.379 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:24.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:24.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 53 16 71033 71017 83.7341 56.125 0.0138863 0.0119356 2024-09-14T04:30:24.531 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:24.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:24.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:24.702 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:24.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:24.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:24.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.846 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:24.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:24.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:24.970 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:24.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:24 smithi001 ceph-mon[107399]: pgmap v1620: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 796 op/s; 27456/201366 objects degraded (13.635%); 6020/201366 objects misplaced (2.990%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:30:24.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/201366 objects degraded (13.635%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:24 smithi001 ceph-mon[112165]: pgmap v1620: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 796 op/s; 27456/201366 objects degraded (13.635%); 6020/201366 objects misplaced (2.990%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:30:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/201366 objects degraded (13.635%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:24.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:25.072 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:24 smithi012 ceph-mon[101180]: pgmap v1620: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 796 op/s; 27456/201366 objects degraded (13.635%); 6020/201366 objects misplaced (2.990%); 3.8 MiB/s, 60 objects/s recovering 2024-09-14T04:30:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/201366 objects degraded (13.635%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:25.102 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:25.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:25.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:25.352 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:25.384 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:25.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 54 16 71831 71815 83.107 49.875 0.0276273 0.0120259 2024-09-14T04:30:25.486 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.521 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:25.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:25.655 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:25.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:25.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:25.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:25.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:25.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:26.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:26.064 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:26.166 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:26.198 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:26.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:26.345 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:26.447 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:26.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 55 16 72701 72685 82.5846 54.375 0.0103161 0.0121027 2024-09-14T04:30:26.477 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:26.580 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:26.614 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:26.716 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:26.745 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:26.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:26.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:26.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:26 smithi001 ceph-mon[107399]: pgmap v1621: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 27456/213333 objects degraded (12.870%); 6020/213333 objects misplaced (2.822%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:30:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:26 smithi001 ceph-mon[112165]: pgmap v1621: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 27456/213333 objects degraded (12.870%); 6020/213333 objects misplaced (2.822%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:30:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:26] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.51.0" 2024-09-14T04:30:26.998 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.030 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:27.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:26 smithi012 ceph-mon[101180]: pgmap v1621: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 27456/213333 objects degraded (12.870%); 6020/213333 objects misplaced (2.822%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:30:27.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:27.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:27.279 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:27.311 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:27.413 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:27.446 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:27.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 56 16 73579 73563 82.0897 54.875 0.0150483 0.0121757 2024-09-14T04:30:27.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:27.598 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:27.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:27.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:27.842 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:27.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:27.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:27.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:27 smithi001 ceph-mon[107399]: pgmap v1622: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 813 op/s; 27456/214947 objects degraded (12.773%); 5800/214947 objects misplaced (2.698%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:27 smithi001 ceph-mon[112165]: pgmap v1622: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 813 op/s; 27456/214947 objects degraded (12.773%); 5800/214947 objects misplaced (2.698%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:27.978 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:28.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:28.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:28.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:27 smithi012 ceph-mon[101180]: pgmap v1622: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 813 op/s; 27456/214947 objects degraded (12.773%); 5800/214947 objects misplaced (2.698%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:28.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:28.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.155 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:28.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:28.256 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:28.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:28.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:28.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:28.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:28.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 57 16 74490 74474 81.6484 56.9375 0.00857246 0.0122416 2024-09-14T04:30:28.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:28.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:28.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:28.670 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:28.708 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:28.775 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:30:28.775 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:30:28.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:30:28.775 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:30:28.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246146 12.1 12:86d285cb:::benchmark_data_smithi001_255951_object100486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 477.96691401099997, 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.70537081599999996, 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.776 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246146 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804095+0000", 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804097+0000", 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2139999999999999e-06 2024-09-14T04:30:28.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804102+0000", 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1560000000000003e-06 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804115+0000", 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3072e-05 2024-09-14T04:30:28.779 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804122+0000", 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.849e-06 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804175+0000", 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3134000000000003e-05 2024-09-14T04:30:28.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.804180+0000", 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5799999999999999e-06 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500297+0000", 2024-09-14T04:30:28.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.69611708299999997 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500422+0000", 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000124814 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.500539+0000", 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011681299999999999 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.782 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503578+0000", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00303915 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503636+0000", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7541999999999999e-05 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509290+0000", 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056538700000000001 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509357+0000", 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6892999999999998e-05 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509466+0000", 2024-09-14T04:30:28.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000109646 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246149 12.1 12:815a4314:::benchmark_data_smithi001_255951_object100489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 477.89082740399999, 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62963798900000001, 2024-09-14T04:30:28.785 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246149 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:30:28.786 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880182+0000", 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880184+0000", 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8500000000000001e-06 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.787 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880191+0000", 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4139999999999997e-06 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880206+0000", 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5213e-05 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880214+0000", 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8699999999999992e-06 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880277+0000", 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2564999999999995e-05 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:30.880289+0000", 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1912e-05 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503667+0000", 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.62337792000000003 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503790+0000", 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012371800000000001 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.503926+0000", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000135828 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509269+0000", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.005342935 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509528+0000", 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025847700000000001 2024-09-14T04:30:28.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509684+0000", 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015632800000000001 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509732+0000", 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7926000000000002e-05 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509820+0000", 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8033000000000002e-05 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.794 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246168 12.1 12:80e4f094:::benchmark_data_smithi001_255951_object100508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 477.74219506499998, 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.48125710599999999, 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246168 2024-09-14T04:30:28.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028814+0000", 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028815+0000", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.15e-06 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028820+0000", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.51e-06 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028839+0000", 2024-09-14T04:30:28.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9276999999999999e-05 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028844+0000", 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8930000000000004e-06 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028906+0000", 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1419000000000003e-05 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.028917+0000", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1065e-05 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506507+0000", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.47759034299999997 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506620+0000", 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011265299999999999 2024-09-14T04:30:28.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.506740+0000", 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120423 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509580+0000", 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.002840371 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509849+0000", 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00026879699999999998 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509915+0000", 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6259000000000004e-05 2024-09-14T04:30:28.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.509976+0000", 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0838999999999998e-05 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510071+0000", 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5106999999999996e-05 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:246175 12.1 12:8f83bc3f:::benchmark_data_smithi001_255951_object100515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 477.68076633700002, 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.42337427, 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:30:28.803 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 246175 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090243+0000", 2024-09-14T04:30:28.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090245+0000", 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.562e-06 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.805 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090251+0000", 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1940000000000003e-06 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090265+0000", 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4567e-05 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090274+0000", 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6070000000000008e-06 2024-09-14T04:30:28.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090339+0000", 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5283999999999994e-05 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.090346+0000", 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6109999999999999e-06 2024-09-14T04:30:28.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510119+0000", 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.41977286000000003 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510221+0000", 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000102538 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:28.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.510288+0000", 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6341999999999996e-05 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513037+0000", 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0027498499999999999 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513154+0000", 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116457 2024-09-14T04:30:28.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513461+0000", 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030725199999999999 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513523+0000", 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1754999999999994e-05 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:22:31.513617+0000", 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.4390999999999998e-05 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34091 14.1 14:8adb7a83:::benchmark_data_smithi001_337422_object34090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e503)", 2024-09-14T04:30:28.811 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:48.946269+0000", 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.824740777000002, 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.008386819, 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.812 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34091 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946269+0000", 2024-09-14T04:30:28.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946269+0000", 2024-09-14T04:30:28.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946270+0000", 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.505e-06 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946290+0000", 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0004999999999999e-05 2024-09-14T04:30:28.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946302+0000", 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1487e-05 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.946308+0000", 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.28e-06 2024-09-14T04:30:28.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.949544+0000", 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0032358629999999998 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.949559+0000", 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5019e-05 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.947671+0000", 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99811180499999996 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.947789+0000", 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011799900000000001 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.818 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.947890+0000", 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000100939 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.950998+0000", 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031088209999999999 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.951040+0000", 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1866000000000002e-05 2024-09-14T04:30:28.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954588+0000", 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0035475419999999999 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954626+0000", 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8522999999999997e-05 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954655+0000", 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9164999999999999e-05 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34128 14.1 14:85327436:::benchmark_data_smithi001_337422_object34127:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:48.969190+0000", 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.801819870999999, 2024-09-14T04:30:28.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98572503499999997, 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.822 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34128 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969190+0000", 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969190+0000", 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969192+0000", 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2539999999999999e-06 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969216+0000", 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4583000000000002e-05 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969281+0000", 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4605999999999998e-05 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.969289+0000", 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7349999999999996e-06 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.970221+0000", 2024-09-14T04:30:28.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00093226599999999995 2024-09-14T04:30:28.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.829 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.970232+0000", 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0874e-05 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.949529+0000", 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97929737299999997 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.949665+0000", 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013603899999999999 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.949812+0000", 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014683799999999999 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.951081+0000", 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0012689349999999999 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954741+0000", 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036602829999999999 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954799+0000", 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8187999999999999e-05 2024-09-14T04:30:28.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954872+0000", 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3015000000000005e-05 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954915+0000", 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2045999999999997e-05 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34138 14.1 14:80a59f4b:::benchmark_data_smithi001_337422_object34137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:48.978290+0000", 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.792719712999997, 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97690179799999999, 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.834 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34138 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978290+0000", 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978290+0000", 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978290+0000", 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9100000000000004e-07 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978313+0000", 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3251000000000002e-05 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978321+0000", 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2459999999999997e-06 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978326+0000", 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9830000000000004e-06 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978765+0000", 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043916199999999999 2024-09-14T04:30:28.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.978775+0000", 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0535e-05 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.951122+0000", 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97234650899999997 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.951254+0000", 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000132566 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.951390+0000", 2024-09-14T04:30:28.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013518800000000001 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954702+0000", 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003312738 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954926+0000", 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00022326799999999999 2024-09-14T04:30:28.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955105+0000", 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00017934799999999999 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955156+0000", 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0729000000000002e-05 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955191+0000", 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5784000000000002e-05 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34160 14.1 14:81a6756c:::benchmark_data_smithi001_337422_object34159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:48.990867+0000", 2024-09-14T04:30:28.842 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.780142458999997, 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96454256299999996, 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34160 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990867+0000", 2024-09-14T04:30:28.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990867+0000", 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990869+0000", 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8449999999999999e-06 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990888+0000", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8816000000000001e-05 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990898+0000", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.013e-05 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990904+0000", 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.224e-06 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990972+0000", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8046000000000005e-05 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.990983+0000", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0730999999999999e-05 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.952573+0000", 2024-09-14T04:30:28.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96159069500000005 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.952700+0000", 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012612499999999999 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.952853+0000", 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000153268 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.954930+0000", 2024-09-14T04:30:28.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0020774890000000001 2024-09-14T04:30:28.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.848 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:28.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955051+0000", 2024-09-14T04:30:28.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012092 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955312+0000", 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00026083500000000001 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955364+0000", 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1669999999999998e-05 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955409+0000", 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5769000000000002e-05 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34169 14.1 14:8286947d:::benchmark_data_smithi001_337422_object34168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:48.998515+0000", 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.772494438999999, 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95708617299999998, 2024-09-14T04:30:28.852 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34169 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998515+0000", 2024-09-14T04:30:28.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998515+0000", 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998516+0000", 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2750000000000001e-06 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998537+0000", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1126999999999999e-05 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998549+0000", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2092999999999999e-05 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998555+0000", 2024-09-14T04:30:28.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7989999999999999e-06 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998631+0000", 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5265999999999994e-05 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:48.998640+0000", 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3719999999999994e-06 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.953721+0000", 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95508127700000001 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.953807+0000", 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6236000000000006e-05 2024-09-14T04:30:28.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.953901+0000", 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3277999999999997e-05 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955203+0000", 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001302683 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955253+0000", 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0059999999999998e-05 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955468+0000", 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021492899999999999 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955542+0000", 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3798000000000001e-05 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955601+0000", 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8980000000000001e-05 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34172 14.1 14:88c766eb:::benchmark_data_smithi001_337422_object34171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.000602+0000", 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.770406882000003, 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.96245892300000002, 2024-09-14T04:30:28.860 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34172 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000602+0000", 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000602+0000", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000604+0000", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6929999999999999e-06 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000627+0000", 2024-09-14T04:30:28.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2739999999999999e-05 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000643+0000", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6328e-05 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000650+0000", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.973e-06 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000711+0000", 2024-09-14T04:30:28.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1174999999999996e-05 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.000721+0000", 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1390000000000001e-06 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955653+0000", 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95493284499999997 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955814+0000", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016105700000000001 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.955967+0000", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015227900000000001 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959644+0000", 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036771410000000001 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959655+0000", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1255e-05 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.962975+0000", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0033197040000000001 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963024+0000", 2024-09-14T04:30:28.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8674000000000002e-05 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963061+0000", 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7920000000000003e-05 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34181 14.1 14:8861ff58:::benchmark_data_smithi001_337422_object34180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.005401+0000", 2024-09-14T04:30:28.867 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.765607903999999, 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95777374599999998, 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34181 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005401+0000", 2024-09-14T04:30:28.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005401+0000", 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005403+0000", 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.801e-06 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005427+0000", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.3904e-05 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005440+0000", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2941e-05 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005446+0000", 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5199999999999997e-06 2024-09-14T04:30:28.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005505+0000", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9481999999999997e-05 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.005512+0000", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6560000000000003e-06 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.957026+0000", 2024-09-14T04:30:28.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95151397699999996 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.957166+0000", 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000140459 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.957308+0000", 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014132000000000001 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959660+0000", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023521639999999999 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959698+0000", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8173999999999997e-05 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963105+0000", 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003406713 2024-09-14T04:30:28.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963144+0000", 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9526000000000001e-05 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963175+0000", 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.1109000000000003e-05 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34182 14.1 14:85c4bd6f:::benchmark_data_smithi001_337422_object34181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.006563+0000", 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.764446681000003, 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95833141200000005, 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34182 2024-09-14T04:30:28.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006563+0000", 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006563+0000", 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006564+0000", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0690000000000001e-06 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006585+0000", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1009000000000002e-05 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006597+0000", 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1956e-05 2024-09-14T04:30:28.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006603+0000", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5540000000000003e-06 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006673+0000", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9926999999999996e-05 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.006684+0000", 2024-09-14T04:30:28.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0334e-05 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.958210+0000", 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95152674100000001 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.958345+0000", 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000134757 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.879 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.958489+0000", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014358400000000001 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.962910+0000", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0044212219999999998 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963219+0000", 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00030884700000000001 2024-09-14T04:30:28.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.964792+0000", 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015737399999999999 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.964850+0000", 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7698999999999998e-05 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.964894+0000", 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3973000000000001e-05 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34193 14.1 14:856dc77e:::benchmark_data_smithi001_337422_object34192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.012613+0000", 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.758395995000001, 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95975912100000005, 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.882 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34193 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012613+0000", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012613+0000", 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012615+0000", 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9750000000000001e-06 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012640+0000", 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.4324999999999998e-05 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012654+0000", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3862999999999999e-05 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012661+0000", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9940000000000003e-06 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012737+0000", 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.6256999999999999e-05 2024-09-14T04:30:28.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.012746+0000", 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.5219999999999996e-06 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959748+0000", 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94700150599999999 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.886 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.959893+0000", 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000145644 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.960035+0000", 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000141595 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963259+0000", 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003224153 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.887 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.964905+0000", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0016457030000000001 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972282+0000", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0073771339999999996 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972335+0000", 2024-09-14T04:30:28.888 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3156999999999999e-05 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972373+0000", 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7292999999999997e-05 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34212 14.1 14:838cb61a:::benchmark_data_smithi001_337422_object34211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.026482+0000", 2024-09-14T04:30:28.889 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.744527599000001, 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94621127800000004, 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34212 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026482+0000", 2024-09-14T04:30:28.890 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026482+0000", 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026484+0000", 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7689999999999999e-06 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026534+0000", 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0012999999999997e-05 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026551+0000", 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7523000000000001e-05 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026560+0000", 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5839999999999995e-06 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026624+0000", 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.4356000000000005e-05 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.026633+0000", 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1060000000000006e-06 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.961214+0000", 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.93458114400000003 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.961346+0000", 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013218600000000001 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.961479+0000", 2024-09-14T04:30:28.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000132652 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.966655+0000", 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051754749999999997 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972564+0000", 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059096950000000004 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972582+0000", 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7847e-05 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972650+0000", 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7923999999999995e-05 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972693+0000", 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3003999999999997e-05 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.896 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34245 14.1 14:8d9eb9ae:::benchmark_data_smithi001_337422_object34244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.051287+0000", 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.719722595, 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.92162664500000002, 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.897 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34245 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051287+0000", 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051287+0000", 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051289+0000", 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8050000000000001e-06 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051310+0000", 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.1861e-05 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.899 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051327+0000", 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6799e-05 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051335+0000", 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5279999999999998e-06 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051404+0000", 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9620000000000001e-05 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.051415+0000", 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0487e-05 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963309+0000", 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.91189375800000005 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963438+0000", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012940400000000001 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.963565+0000", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012669299999999999 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.966698+0000", 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031329830000000002 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972749+0000", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0060512700000000001 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972807+0000", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8199999999999998e-05 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972875+0000", 2024-09-14T04:30:28.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.7347000000000004e-05 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972913+0000", 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8890000000000002e-05 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34260 14.1 14:81d23683:::benchmark_data_smithi001_337422_object34259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.904 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.064203+0000", 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.706806489999998, 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90890552700000005, 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34260 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064203+0000", 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064203+0000", 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064205+0000", 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.719e-06 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064230+0000", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5761e-05 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064243+0000", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2228e-05 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064251+0000", 2024-09-14T04:30:28.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9459999999999998e-06 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064326+0000", 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5772999999999994e-05 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064338+0000", 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1756000000000001e-05 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.964941+0000", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90060298400000005 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.965073+0000", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000131772 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.965216+0000", 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000143363 2024-09-14T04:30:28.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972498+0000", 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0072820289999999998 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972755+0000", 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000256479 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973011+0000", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00025589500000000002 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973064+0000", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3535000000000002e-05 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973108+0000", 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4286999999999998e-05 2024-09-14T04:30:28.911 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34261 14.1 14:816f457c:::benchmark_data_smithi001_337422_object34260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.064815+0000", 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.706194275999998, 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90846881800000001, 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.912 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34261 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064815+0000", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064815+0000", 2024-09-14T04:30:28.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064816+0000", 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2399999999999997e-07 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064847+0000", 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0893999999999999e-05 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.914 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064858+0000", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0878e-05 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064865+0000", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.3660000000000003e-06 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064924+0000", 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8956999999999998e-05 2024-09-14T04:30:28.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.064933+0000", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3999999999999998e-06 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.966735+0000", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90180108800000003 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.966880+0000", 2024-09-14T04:30:28.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014520099999999999 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.967032+0000", 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000152721 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972738+0000", 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0057051599999999999 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972760+0000", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2189e-05 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973163+0000", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000403614 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973244+0000", 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0446999999999997e-05 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973284+0000", 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0079e-05 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34270 14.1 14:8229af57:::benchmark_data_smithi001_337422_object34269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.073312+0000", 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.697697150000003, 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.90011796099999997, 2024-09-14T04:30:28.919 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34270 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073312+0000", 2024-09-14T04:30:28.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073312+0000", 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073314+0000", 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.917e-06 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073340+0000", 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5873000000000001e-05 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073355+0000", 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4484999999999999e-05 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073362+0000", 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7870000000000002e-06 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073430+0000", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8207999999999995e-05 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.073440+0000", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.1619999999999997e-06 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.968677+0000", 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.89523713199999999 2024-09-14T04:30:28.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.968809+0000", 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013184899999999999 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.968978+0000", 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016916500000000001 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972765+0000", 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.003786719 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.972960+0000", 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019584 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973335+0000", 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00037485699999999998 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973397+0000", 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.1556999999999994e-05 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973430+0000", 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.341e-05 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34278 14.1 14:8c67d72b:::benchmark_data_smithi001_337422_object34277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.085980+0000", 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.685029135999997, 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.89916087300000003, 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34278 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.927 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.085980+0000", 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.085980+0000", 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.928 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.085983+0000", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.6060000000000001e-06 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086016+0000", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2805999999999997e-05 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086038+0000", 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.2589999999999999e-05 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086050+0000", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1474000000000001e-05 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086147+0000", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7765999999999999e-05 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086160+0000", 2024-09-14T04:30:28.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2418999999999999e-05 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973466+0000", 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.88730632899999995 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973585+0000", 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000118347 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.973726+0000", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000141149 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.980857+0000", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0071315809999999997 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.981037+0000", 2024-09-14T04:30:28.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000179832 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.984845+0000", 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0038078980000000001 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.985091+0000", 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024618900000000001 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.985141+0000", 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9886999999999999e-05 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:34279 14.1 14:8da59f9f:::benchmark_data_smithi001_337422_object34278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:29:49.086958+0000", 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 39.684051642, 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.89828083700000005, 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:30:28.934 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 34279 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086958+0000", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:30:28.935 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086958+0000", 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086959+0000", 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.9900000000000009e-07 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086977+0000", 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7816999999999999e-05 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086987+0000", 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0625000000000001e-05 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.086993+0000", 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4450000000000004e-06 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.087044+0000", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1610999999999998e-05 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.087050+0000", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8139999999999997e-06 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.975074+0000", 2024-09-14T04:30:28.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.88802426400000001 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.975182+0000", 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010783 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.975292+0000", 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010983 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.980914+0000", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0056224109999999999 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.981112+0000", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00019774500000000001 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.985158+0000", 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0040454389999999996 2024-09-14T04:30:28.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.985202+0000", 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4746000000000002e-05 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:29:49.985239+0000", 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6260999999999997e-05 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.942 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:30:28.943 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:30:28.943 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:30:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:28.943 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:28.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:29.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:29.081 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:29.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:29.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:29.333 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:29.367 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:29.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 58 16 75381 75365 81.2007 55.6875 0.022263 0.0123099 2024-09-14T04:30:29.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:29.514 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:29.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:29.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/214947 objects degraded (12.773%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:29.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:29.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:29.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/214947 objects degraded (12.773%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:29.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/214947 objects degraded (12.773%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:29.748 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:30:29.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:29.808 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:29.911 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:29.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:29.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:30.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:30.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:30.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:30:30.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:30.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:30.384 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.426 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:30.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:30.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 59 16 76218 76202 80.711 52.3125 0.0081889 0.0123835 2024-09-14T04:30:30.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:30 smithi001 ceph-mon[107399]: pgmap v1623: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 27456/215724 objects degraded (12.727%); 5800/215724 objects misplaced (2.689%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:30.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:30 smithi001 ceph-mon[112165]: pgmap v1623: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 27456/215724 objects degraded (12.727%); 5800/215724 objects misplaced (2.689%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:30.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:30.554 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:30.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:30.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:30 smithi012 ceph-mon[101180]: pgmap v1623: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 27456/215724 objects degraded (12.727%); 5800/215724 objects misplaced (2.689%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:30.606 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:30.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.709 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:30.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:30.810 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:30.845 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:30.947 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:30.976 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:30.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:30.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:31.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:31.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:31.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:31.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:31.259 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:31.361 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:31.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:31.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:30:31.448665+0000 min lat: 0.00130141 max lat: 2.15405 avg lat: 0.0124608 2024-09-14T04:30:31.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:30:31.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 60 15 77025 77010 80.2074 50.5 0.0266727 0.0124608 2024-09-14T04:30:31.499 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:31.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:31.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:31.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:31.752 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:31.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:31.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:31.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:31.916 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:31.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:31.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:32.057 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:32.091 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:32.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:32.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:32.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:32.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:32.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:32.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:32.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 61 16 77877 77861 79.7644 53.1875 0.0228699 0.0125283 2024-09-14T04:30:32.484 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:32.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:32.615 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:32.647 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:32.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:32.665 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:30:32.749 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:32.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:32.817 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:30:32.817 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:30:32.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.817 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78137 14.b e516/510 14:da500612:::benchmark_data_smithi001_337422_object78136:head v 516'4942, mlcod=0'0)", 2024-09-14T04:30:32.817 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.812172+0000", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0017509139999999999, 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017798740000000001, 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812172+0000", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812172+0000", 2024-09-14T04:30:32.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812172+0000", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812209+0000", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.735e-05 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812217+0000", 2024-09-14T04:30:32.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3119999999999994e-06 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812223+0000", 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0059999999999998e-06 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812527+0000", 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030384200000000002 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812549+0000", 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2177000000000001e-05 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78139 14.1 e516/510)", 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.813903+0000", 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.9791000000000001e-05, 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111903, 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:30:32.821 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813903+0000", 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813903+0000", 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813902+0000", 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813933+0000", 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1380000000000001e-05 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813944+0000", 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0949e-05 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813951+0000", 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3219999999999998e-06 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78119 14.3 e516/510 14:c5360700:::benchmark_data_smithi001_337422_object78118:head v 516'4847, mlcod=0'0)", 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.793320+0000", 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.020603403999999999, 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020742545000000001, 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.824 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793320+0000", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793320+0000", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793319+0000", 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793356+0000", 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6992000000000001e-05 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793363+0000", 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7850000000000003e-06 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793367+0000", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4750000000000004e-06 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793407+0000", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9227000000000002e-05 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.796576+0000", 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031693929999999999 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.827 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:78122 14.e 14:710caae4:::benchmark_data_smithi001_337422_object78121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.793355+0000", 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.020567917000000002, 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020770097000000001, 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:32.828 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 78122 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793355+0000", 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793355+0000", 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793355+0000", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793393+0000", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8758999999999999e-05 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793401+0000", 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0919999999999998e-06 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793407+0000", 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7960000000000001e-06 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793451+0000", 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4039999999999998e-05 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793663+0000", 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0002118 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.793856+0000", 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019269999999999999 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:32.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.796700+0000", 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028440119999999999 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812266+0000", 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015565834000000001 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812348+0000", 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1902000000000003e-05 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812625+0000", 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027758600000000002 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78127 14.b e516/510 14:d60edd0d:::benchmark_data_smithi001_337422_object78126:head v 516'4941, mlcod=0'0)", 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.798922+0000", 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.015000813999999999, 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015295654000000001, 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.834 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798922+0000", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798922+0000", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798922+0000", 2024-09-14T04:30:32.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798973+0000", 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1177000000000001e-05 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798982+0000", 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0019999999999995e-06 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.798988+0000", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9719999999999997e-06 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.799330+0000", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034266400000000001 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.799352+0000", 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1602000000000001e-05 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.837 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78131 14.3 e516/510 14:cc1be574:::benchmark_data_smithi001_337422_object78130:head v 516'4848, mlcod=0'0)", 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.804722+0000", 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.009200761, 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009550757, 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804722+0000", 2024-09-14T04:30:32.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804722+0000", 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804721+0000", 2024-09-14T04:30:32.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804755+0000", 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3433000000000003e-05 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804765+0000", 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0754000000000001e-05 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.804771+0000", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0020000000000003e-06 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.810290+0000", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005518121 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.810342+0000", 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2027000000000002e-05 2024-09-14T04:30:32.841 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:78134 14.1 e516/510 14:8fa4431a:::benchmark_data_smithi001_337422_object78133:head v 516'4936, mlcod=503'1261)", 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.809068+0000", 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0048546099999999997, 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052611020000000001, 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809068+0000", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809068+0000", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809068+0000", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809107+0000", 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9452000000000001e-05 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809334+0000", 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022668300000000001 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809341+0000", 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8589999999999999e-06 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809767+0000", 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000426069 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809789+0000", 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2288000000000001e-05 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.845 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:78135 14.7 14:ea14cbe2:::benchmark_data_smithi001_337422_object78134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.809280+0000", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0046430589999999997, 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051112680000000004, 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 78135 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809280+0000", 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809280+0000", 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809278+0000", 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809310+0000", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1378999999999999e-05 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809378+0000", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7848999999999994e-05 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809385+0000", 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4329999999999998e-06 2024-09-14T04:30:32.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809461+0000", 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6064999999999998e-05 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809783+0000", 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032197999999999998 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:30:32.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.809915+0000", 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013219600000000001 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.813269+0000", 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033542009999999998 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.46352.0:78122 14.e e516/510 ondisk, result = 0)", 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:32.812129+0000", 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0017940740000000001, 2024-09-14T04:30:32.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023349199999999999, 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812129+0000", 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812129+0000", 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812128+0000", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812135+0000", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0639999999999996e-06 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812152+0000", 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6622000000000001e-05 2024-09-14T04:30:32.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812159+0000", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7829999999999999e-06 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812293+0000", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133578 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:32.812328+0000", 2024-09-14T04:30:32.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5228000000000002e-05 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 9 2024-09-14T04:30:32.854 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:30:32.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:32.931 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:32.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:32.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[112165]: pgmap v1624: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 27456/226287 objects degraded (12.133%); 5800/226287 objects misplaced (2.563%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1390972949' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[107399]: pgmap v1624: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 27456/226287 objects degraded (12.133%); 5800/226287 objects misplaced (2.563%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1390972949' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:32.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:32 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:33.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:33.076 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:33.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:32 smithi012 ceph-mon[101180]: pgmap v1624: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 27456/226287 objects degraded (12.133%); 5800/226287 objects misplaced (2.563%); 2.3 MiB/s, 37 objects/s recovering 2024-09-14T04:30:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1390972949' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:32 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:30:33.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:33.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:33.313 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:33.343 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:33.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:33.448 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 62 16 78639 78623 79.246 47.625 0.071133 0.0126113 2024-09-14T04:30:33.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.490 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:33.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:33.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:33.627 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:33.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:30:33.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:33.819 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:33.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:30:33.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:30:33.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[107399]: osdmap e516: 8 total, 8 up, 6 in 2024-09-14T04:30:33.914 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:30:33.914 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:30:33.914 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:33 smithi001 ceph-mon[112165]: osdmap e516: 8 total, 8 up, 6 in 2024-09-14T04:30:33.921 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:33.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:33.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:34.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:33 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:30:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:33 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:30:34.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:33 smithi012 ceph-mon[101180]: osdmap e516: 8 total, 8 up, 6 in 2024-09-14T04:30:34.120 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:34.124 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:34.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:34.254 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:34.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:34.391 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:34.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 63 16 79458 79442 78.8005 51.1875 0.0162994 0.0126844 2024-09-14T04:30:34.492 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:34.522 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:34.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:34.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:34.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:34.794 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:34.896 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:34.929 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:34.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:34.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:34 smithi001 ceph-mon[107399]: pgmap v1626: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 887 op/s; 27456/227985 objects degraded (12.043%); 5576/227985 objects misplaced (2.446%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:34.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/227985 objects degraded (12.043%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:34 smithi001 ceph-mon[112165]: pgmap v1626: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 887 op/s; 27456/227985 objects degraded (12.043%); 5576/227985 objects misplaced (2.446%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/227985 objects degraded (12.043%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:35.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:35.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:34 smithi012 ceph-mon[101180]: pgmap v1626: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 55 MiB/s wr, 887 op/s; 27456/227985 objects degraded (12.043%); 5576/227985 objects misplaced (2.446%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/227985 objects degraded (12.043%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:35.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:35.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:35.314 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:35.346 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:35.448 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:35.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 64 16 80309 80293 78.4003 53.1875 0.0112949 0.0127485 2024-09-14T04:30:35.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:35.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:35.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:35.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.791 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:35.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:35.852 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:30:35.852 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:30:35.880 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:30:35.880 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:30:35.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:35.908 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:30:35.909 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:30:35.934 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:30:35.935 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:30:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:35.936 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:35.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:35.963 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:30:35.964 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:30:35.998 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:30:35.998 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:30:36.033 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:30:36.033 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:30:36.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:36.060 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:30:36.061 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:30:36.061 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:36.061 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:36.061 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[107399]: pgmap v1627: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 27456/236367 objects degraded (11.616%); 5576/236367 objects misplaced (2.359%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:36.061 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2434384967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:36.062 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:36.062 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:36.062 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[112165]: pgmap v1627: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 27456/236367 objects degraded (11.616%); 5576/236367 objects misplaced (2.359%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:36.062 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2434384967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:36.069 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:35 smithi012 ceph-mon[101180]: pgmap v1627: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 27456/236367 objects degraded (11.616%); 5576/236367 objects misplaced (2.359%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:30:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2434384967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:36.151 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 7, 6] out_osds: [2, 1] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:30:36.152 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:30:36.152 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2024-09-14T04:30:36.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:30:36.172 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:36.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:36.330 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:30:36.330 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:30:36.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:36.378 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:36.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 65 16 81116 81100 77.9701 50.4375 0.00941394 0.0128178 2024-09-14T04:30:36.480 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:36.510 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:36.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:36.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:36.720 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:36] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.51.0" 2024-09-14T04:30:36.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:36.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:36.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:36.917 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:36.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:36.999 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:37.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.068 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:37.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:37.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:30:37.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:37.233 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:37.251 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:30:37.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:37.379 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:37.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:37.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 66 16 81935 81919 77.5642 51.1875 0.0331086 0.0128863 2024-09-14T04:30:37.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:37.535 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:37.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:37.612 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:37.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.711 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:37.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:37.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:37.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:37.855 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:37.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:37.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:38.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:38.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:38.141 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:38.243 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:38.273 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:38.375 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:38.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:38.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 67 16 82678 82662 77.0996 46.4375 0.0170143 0.0129635 2024-09-14T04:30:38.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:38.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:38.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:38.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:38.695 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:38.723 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:38 smithi012 ceph-mon[101180]: pgmap v1628: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 27456/240204 objects degraded (11.430%); 5334/240204 objects misplaced (2.221%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:38.740 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:38.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:38.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:38.913 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:30:38.940 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:38 smithi001 ceph-mon[107399]: pgmap v1628: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 27456/240204 objects degraded (11.430%); 5334/240204 objects misplaced (2.221%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:38.940 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:38 smithi001 ceph-mon[112165]: pgmap v1628: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s; 27456/240204 objects degraded (11.430%); 5334/240204 objects misplaced (2.221%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:38.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:38.980 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:38.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:39.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.130 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:39.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:39.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:39.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:39.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:39.394 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:39.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 68 16 83380 83364 76.611 43.875 0.0608685 0.0130463 2024-09-14T04:30:39.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:39.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:39.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:39.669 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:39.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:39.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:39.799 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:39.901 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:39.932 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:39.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:39.934 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/240204 objects degraded (11.430%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:39.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/240204 objects degraded (11.430%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:39.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/240204 objects degraded (11.430%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:30:40.035 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.090 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:40.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:40.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:40.230 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:40.332 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:40.377 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:40.380 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:40.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 69 16 84117 84101 76.1682 46.0625 0.0213643 0.0131221 2024-09-14T04:30:40.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.514 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:40.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:40.599 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:30:40.599 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:30:40.614 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:40.631 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:30:40.632 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:30:40.662 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:40.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.804 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:40.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:40.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:40 smithi012 ceph-mon[101180]: pgmap v1629: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/240939 objects degraded (11.395%); 5334/240939 objects misplaced (2.214%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:40.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:40.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:40.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:40.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:40.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:40 smithi001 ceph-mon[107399]: pgmap v1629: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/240939 objects degraded (11.395%); 5334/240939 objects misplaced (2.214%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:40 smithi001 ceph-mon[112165]: pgmap v1629: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s; 27456/240939 objects degraded (11.395%); 5334/240939 objects misplaced (2.214%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:41.054 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:41.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:41.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:41.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:41.234 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:41.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.376 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:41.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:41.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 70 16 84914 84898 75.7917 49.8125 0.0291647 0.0131872 2024-09-14T04:30:41.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:30:41.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:41.554 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:41.657 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:41.690 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:41.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:41.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:41.852 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:41.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:41.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:41 smithi001 ceph-mon[107399]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:30:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:41 smithi001 ceph-mon[112165]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:30:41.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:41.982 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:41.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:42.005 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:41 smithi012 ceph-mon[101180]: from='client.55834 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:30:42.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:42.116 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1629,"stamp":"2024-09-14T04:30:39.599063+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5262302663,"num_objects":80313,"num_object_clones":0,"num_object_copies":240939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27456,"num_objects_misplaced":5334,"num_objects_unfound":0,"num_objects_dirty":80313,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":80357,"num_write_kb":5138474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5745,"num_bytes_recovered":373502480,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82875,"ondisk_log_size":82875,"up":327,"acting":317,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19808660,"kb_used_data":13293212,"kb_used_omap":25,"kb_used_meta":6515366,"kb_avail":542662252,"statfs":{"total":575970213888,"available":555686146048,"internally_reserved":0,"allocated":13612249088,"data_stored":13591958810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26477,"internal_metadata":6671734931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,5,7,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":550830080,"num_objects":8405,"num_object_clones":0,"num_object_copies":25215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-466,"num_objects_unfound":0,"num_objects_dirty":8405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8405,"num_write_kb":537920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":466,"num_bytes_recovered":30539776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003180"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":837,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678336+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:30:32.678336+0000","last_peered":"2024-09-14T04:30:32.678336+0000","last_clean":"2024-09-14T04:30:32.678336+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:30:32.678336+0000","last_undegraded":"2024-09-14T04:30:32.678336+0000","last_fullsized":"2024-09-14T04:30:32.678336+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":208,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300807+0000","last_change":"2024-09-14T04:30:04.281338+0000","last_active":"2024-09-14T04:30:10.300807+0000","last_peered":"2024-09-14T04:30:10.300807+0000","last_clean":"2024-09-14T04:30:10.300807+0000","last_became_active":"2024-09-14T04:30:04.280363+0000","last_became_peered":"2024-09-14T04:30:04.280363+0000","last_unstale":"2024-09-14T04:30:10.300807+0000","last_undegraded":"2024-09-14T04:30:10.300807+0000","last_fullsized":"2024-09-14T04:30:10.300807+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:50:56.470339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":107,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.301002+0000","last_change":"2024-09-14T04:29:37.998593+0000","last_active":"2024-09-14T04:30:10.301002+0000","last_peered":"2024-09-14T04:30:10.301002+0000","last_clean":"2024-09-14T04:30:10.301002+0000","last_became_active":"2024-09-14T04:29:37.991857+0000","last_became_peered":"2024-09-14T04:29:37.991857+0000","last_unstale":"2024-09-14T04:30:10.301002+0000","last_undegraded":"2024-09-14T04:30:10.301002+0000","last_fullsized":"2024-09-14T04:30:10.301002+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:44:08.110065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":817,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299062+0000","last_change":"2024-09-14T04:29:18.670950+0000","last_active":"2024-09-14T04:30:10.299062+0000","last_peered":"2024-09-14T04:30:10.299062+0000","last_clean":"2024-09-14T04:30:10.299062+0000","last_became_active":"2024-09-14T04:29:18.670619+0000","last_became_peered":"2024-09-14T04:29:18.670619+0000","last_unstale":"2024-09-14T04:30:10.299062+0000","last_undegraded":"2024-09-14T04:30:10.299062+0000","last_fullsized":"2024-09-14T04:30:10.299062+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:02:43.218879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302583+0000","last_change":"2024-09-14T04:29:18.666490+0000","last_active":"2024-09-14T04:30:10.302583+0000","last_peered":"2024-09-14T04:30:10.302583+0000","last_clean":"2024-09-14T04:30:10.302583+0000","last_became_active":"2024-09-14T04:29:18.666113+0000","last_became_peered":"2024-09-14T04:29:18.666113+0000","last_unstale":"2024-09-14T04:30:10.302583+0000","last_undegraded":"2024-09-14T04:30:10.302583+0000","last_fullsized":"2024-09-14T04:30:10.302583+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:30.604568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":660,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678970+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:30:32.678970+0000","last_peered":"2024-09-14T04:30:32.678970+0000","last_clean":"2024-09-14T04:30:32.678970+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:30:32.678970+0000","last_undegraded":"2024-09-14T04:30:32.678970+0000","last_fullsized":"2024-09-14T04:30:32.678970+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":936,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.676240+0000","last_change":"2024-09-14T04:30:07.304986+0000","last_active":"2024-09-14T04:30:32.676240+0000","last_peered":"2024-09-14T04:30:32.676240+0000","last_clean":"2024-09-14T04:30:32.676240+0000","last_became_active":"2024-09-14T04:30:07.304452+0000","last_became_peered":"2024-09-14T04:30:07.304452+0000","last_unstale":"2024-09-14T04:30:32.676240+0000","last_undegraded":"2024-09-14T04:30:32.676240+0000","last_fullsized":"2024-09-14T04:30:32.676240+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:59.521970+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":646,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.683571+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:30:32.683571+0000","last_peered":"2024-09-14T04:30:32.683571+0000","last_clean":"2024-09-14T04:30:32.683571+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:30:32.683571+0000","last_undegraded":"2024-09-14T04:30:32.683571+0000","last_fullsized":"2024-09-14T04:30:32.683571+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.298806+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:30:10.298806+0000","last_peered":"2024-09-14T04:30:10.298806+0000","last_clean":"2024-09-14T04:30:10.298806+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:30:10.298806+0000","last_undegraded":"2024-09-14T04:30:10.298806+0000","last_fullsized":"2024-09-14T04:30:10.298806+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":673,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.309912+0000","last_change":"2024-09-14T04:30:07.297746+0000","last_active":"2024-09-14T04:30:10.309912+0000","last_peered":"2024-09-14T04:30:10.309912+0000","last_clean":"2024-09-14T04:30:10.309912+0000","last_became_active":"2024-09-14T04:30:07.297186+0000","last_became_peered":"2024-09-14T04:30:07.297186+0000","last_unstale":"2024-09-14T04:30:10.309912+0000","last_undegraded":"2024-09-14T04:30:10.309912+0000","last_fullsized":"2024-09-14T04:30:10.309912+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:59:40.613090+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.675387+0000","last_change":"2024-09-14T04:30:04.286496+0000","last_active":"2024-09-14T04:30:32.675387+0000","last_peered":"2024-09-14T04:30:32.675387+0000","last_clean":"2024-09-14T04:30:32.675387+0000","last_became_active":"2024-09-14T04:30:04.282753+0000","last_became_peered":"2024-09-14T04:30:04.282753+0000","last_unstale":"2024-09-14T04:30:32.675387+0000","last_undegraded":"2024-09-14T04:30:32.675387+0000","last_fullsized":"2024-09-14T04:30:32.675387+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:39.799809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":76,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.309244+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:30:10.309244+0000","last_peered":"2024-09-14T04:30:10.309244+0000","last_clean":"2024-09-14T04:30:10.309244+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:30:10.309244+0000","last_undegraded":"2024-09-14T04:30:10.309244+0000","last_fullsized":"2024-09-14T04:30:10.309244+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"516'5041","reported_seq":5080,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:35.156226+0000","last_change":"2024-09-14T04:30:05.463617+0000","last_active":"2024-09-14T04:30:35.156226+0000","last_peered":"2024-09-14T04:30:35.156226+0000","last_clean":"2024-09-14T04:30:03.658952+0000","last_became_active":"2024-09-14T04:30:05.453212+0000","last_became_peered":"2024-09-14T04:30:05.453212+0000","last_unstale":"2024-09-14T04:30:35.156226+0000","last_undegraded":"2024-09-14T04:30:05.292087+0000","last_fullsized":"2024-09-14T04:30:05.271665+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:22.421071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330366976,"num_objects":5041,"num_object_clones":0,"num_object_copies":15123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5041,"num_write_kb":322624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":3407},{"shards":"3,6","objects":1634}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":802,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302774+0000","last_change":"2024-09-14T04:30:07.303908+0000","last_active":"2024-09-14T04:30:10.302774+0000","last_peered":"2024-09-14T04:30:10.302774+0000","last_clean":"2024-09-14T04:30:10.302774+0000","last_became_active":"2024-09-14T04:30:07.302591+0000","last_became_peered":"2024-09-14T04:30:07.302591+0000","last_unstale":"2024-09-14T04:30:10.302774+0000","last_undegraded":"2024-09-14T04:30:10.302774+0000","last_fullsized":"2024-09-14T04:30:10.302774+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:51.333058+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":90,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.301190+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:30:10.301190+0000","last_peered":"2024-09-14T04:30:10.301190+0000","last_clean":"2024-09-14T04:30:10.301190+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:30:10.301190+0000","last_undegraded":"2024-09-14T04:30:10.301190+0000","last_fullsized":"2024-09-14T04:30:10.301190+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":88,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299210+0000","last_change":"2024-09-14T04:29:18.666721+0000","last_active":"2024-09-14T04:30:10.299210+0000","last_peered":"2024-09-14T04:30:10.299210+0000","last_clean":"2024-09-14T04:30:10.299210+0000","last_became_active":"2024-09-14T04:29:18.666442+0000","last_became_peered":"2024-09-14T04:29:18.666442+0000","last_unstale":"2024-09-14T04:30:10.299210+0000","last_undegraded":"2024-09-14T04:30:10.299210+0000","last_fullsized":"2024-09-14T04:30:10.299210+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:11:52.910370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"516'4927","reported_seq":4984,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.614875+0000","last_change":"2024-09-14T04:30:05.533798+0000","last_active":"2024-09-14T04:30:34.614875+0000","last_peered":"2024-09-14T04:30:34.614875+0000","last_clean":"2024-09-14T04:30:03.664315+0000","last_became_active":"2024-09-14T04:30:05.526576+0000","last_became_peered":"2024-09-14T04:30:05.526576+0000","last_unstale":"2024-09-14T04:30:34.614875+0000","last_undegraded":"2024-09-14T04:30:05.314967+0000","last_fullsized":"2024-09-14T04:30:05.271852+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322895872,"num_objects":4927,"num_object_clones":0,"num_object_copies":14781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3375,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4927,"num_write_kb":315328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":3375},{"shards":"0,3","objects":1552}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1194,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.677016+0000","last_change":"2024-09-14T04:28:22.698300+0000","last_active":"2024-09-14T04:30:32.677016+0000","last_peered":"2024-09-14T04:30:32.677016+0000","last_clean":"2024-09-14T04:30:32.677016+0000","last_became_active":"2024-09-14T04:28:22.697894+0000","last_became_peered":"2024-09-14T04:28:22.697894+0000","last_unstale":"2024-09-14T04:30:32.677016+0000","last_undegraded":"2024-09-14T04:30:32.677016+0000","last_fullsized":"2024-09-14T04:30:32.677016+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:19.688955+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":103,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300497+0000","last_change":"2024-09-14T04:30:04.302042+0000","last_active":"2024-09-14T04:30:10.300497+0000","last_peered":"2024-09-14T04:30:10.300497+0000","last_clean":"2024-09-14T04:30:10.300497+0000","last_became_active":"2024-09-14T04:30:04.281337+0000","last_became_peered":"2024-09-14T04:30:04.281337+0000","last_unstale":"2024-09-14T04:30:10.300497+0000","last_undegraded":"2024-09-14T04:30:10.300497+0000","last_fullsized":"2024-09-14T04:30:10.300497+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:39.377188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":553,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:25.545548+0000","last_change":"2024-09-14T04:30:04.299285+0000","last_active":"2024-09-14T04:30:25.545548+0000","last_peered":"2024-09-14T04:30:25.545548+0000","last_clean":"2024-09-14T04:30:25.545548+0000","last_became_active":"2024-09-14T04:30:04.298894+0000","last_became_peered":"2024-09-14T04:30:04.298894+0000","last_unstale":"2024-09-14T04:30:25.545548+0000","last_undegraded":"2024-09-14T04:30:25.545548+0000","last_fullsized":"2024-09-14T04:30:25.545548+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:24.205159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"516'4844","reported_seq":4905,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.629286+0000","last_change":"2024-09-14T04:29:39.122957+0000","last_active":"2024-09-14T04:30:34.629286+0000","last_peered":"2024-09-14T04:30:34.629286+0000","last_clean":"2024-09-14T04:29:36.976847+0000","last_became_active":"2024-09-14T04:29:39.116541+0000","last_became_peered":"2024-09-14T04:29:39.116541+0000","last_unstale":"2024-09-14T04:30:34.629286+0000","last_undegraded":"2024-09-14T04:29:39.038576+0000","last_fullsized":"2024-09-14T04:29:38.987798+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317456384,"num_objects":4844,"num_object_clones":0,"num_object_copies":14532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":971,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":310016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":971},{"shards":"0,5","objects":3873}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":465,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.310258+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:30:10.310258+0000","last_peered":"2024-09-14T04:30:10.310258+0000","last_clean":"2024-09-14T04:30:10.310258+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:30:10.310258+0000","last_undegraded":"2024-09-14T04:30:10.310258+0000","last_fullsized":"2024-09-14T04:30:10.310258+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":103,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302864+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:30:10.302864+0000","last_peered":"2024-09-14T04:30:10.302864+0000","last_clean":"2024-09-14T04:30:10.302864+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:30:10.302864+0000","last_undegraded":"2024-09-14T04:30:10.302864+0000","last_fullsized":"2024-09-14T04:30:10.302864+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":491,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299781+0000","last_change":"2024-09-14T04:30:04.282704+0000","last_active":"2024-09-14T04:30:10.299781+0000","last_peered":"2024-09-14T04:30:10.299781+0000","last_clean":"2024-09-14T04:30:10.299781+0000","last_became_active":"2024-09-14T04:30:04.282427+0000","last_became_peered":"2024-09-14T04:30:04.282427+0000","last_unstale":"2024-09-14T04:30:10.299781+0000","last_undegraded":"2024-09-14T04:30:10.299781+0000","last_fullsized":"2024-09-14T04:30:10.299781+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:04.930405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"516'5120","reported_seq":10189,"reported_epoch":516,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:30:36.930956+0000","last_change":"2024-09-14T04:30:05.563793+0000","last_active":"2024-09-14T04:30:36.930956+0000","last_peered":"2024-09-14T04:30:36.930956+0000","last_clean":"2024-09-14T04:30:03.660024+0000","last_became_active":"2024-09-14T04:30:05.525937+0000","last_became_peered":"2024-09-14T04:30:05.525937+0000","last_unstale":"2024-09-14T04:30:36.930956+0000","last_undegraded":"2024-09-14T04:30:36.930956+0000","last_fullsized":"2024-09-14T04:30:05.272711+0000","mapping_epoch":510,"log_start":"498'300","ondisk_log_start":"498'300","created":494,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4820,"log_dups_size":0,"ondisk_log_size":4820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335544320,"num_objects":5120,"num_object_clones":0,"num_object_copies":15360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1980,"num_objects_unfound":0,"num_objects_dirty":5120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5120,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2499,"num_bytes_recovered":163774464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6],"avail_no_missing":["3","1","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1980},{"shards":"3,6","objects":3140}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":523,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300245+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:30:10.300245+0000","last_peered":"2024-09-14T04:30:10.300245+0000","last_clean":"2024-09-14T04:30:10.300245+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:30:10.300245+0000","last_undegraded":"2024-09-14T04:30:10.300245+0000","last_fullsized":"2024-09-14T04:30:10.300245+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":478,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.674871+0000","last_change":"2024-09-14T04:29:18.673642+0000","last_active":"2024-09-14T04:30:32.674871+0000","last_peered":"2024-09-14T04:30:32.674871+0000","last_clean":"2024-09-14T04:30:32.674871+0000","last_became_active":"2024-09-14T04:29:18.673365+0000","last_became_peered":"2024-09-14T04:29:18.673365+0000","last_unstale":"2024-09-14T04:30:32.674871+0000","last_undegraded":"2024-09-14T04:30:32.674871+0000","last_fullsized":"2024-09-14T04:30:32.674871+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:32.707692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":639,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.689127+0000","last_change":"2024-09-14T04:30:04.292857+0000","last_active":"2024-09-14T04:30:32.689127+0000","last_peered":"2024-09-14T04:30:32.689127+0000","last_clean":"2024-09-14T04:30:32.689127+0000","last_became_active":"2024-09-14T04:30:04.292434+0000","last_became_peered":"2024-09-14T04:30:04.292434+0000","last_unstale":"2024-09-14T04:30:32.689127+0000","last_undegraded":"2024-09-14T04:30:32.689127+0000","last_fullsized":"2024-09-14T04:30:32.689127+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:28.993975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"516'5072","reported_seq":5135,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.702255+0000","last_change":"2024-09-14T04:30:05.487462+0000","last_active":"2024-09-14T04:30:34.702255+0000","last_peered":"2024-09-14T04:30:34.702255+0000","last_clean":"2024-09-14T04:29:36.971690+0000","last_became_active":"2024-09-14T04:30:05.481214+0000","last_became_peered":"2024-09-14T04:30:05.481214+0000","last_unstale":"2024-09-14T04:30:34.702255+0000","last_undegraded":"2024-09-14T04:30:05.375179+0000","last_fullsized":"2024-09-14T04:30:05.271923+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332398592,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4413,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5072,"num_write_kb":324608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1010},{"shards":"0,1,2,3","objects":1},{"shards":"0,1,3","objects":2391},{"shards":"0,3","objects":1669}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":869,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299430+0000","last_change":"2024-09-14T04:29:40.287295+0000","last_active":"2024-09-14T04:30:10.299430+0000","last_peered":"2024-09-14T04:30:10.299430+0000","last_clean":"2024-09-14T04:30:10.299430+0000","last_became_active":"2024-09-14T04:29:40.286891+0000","last_became_peered":"2024-09-14T04:29:40.286891+0000","last_unstale":"2024-09-14T04:30:10.299430+0000","last_undegraded":"2024-09-14T04:30:10.299430+0000","last_fullsized":"2024-09-14T04:30:10.299430+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:59.160582+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":636,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.688687+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:30:32.688687+0000","last_peered":"2024-09-14T04:30:32.688687+0000","last_clean":"2024-09-14T04:30:32.688687+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:30:32.688687+0000","last_undegraded":"2024-09-14T04:30:32.688687+0000","last_fullsized":"2024-09-14T04:30:32.688687+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":546,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299835+0000","last_change":"2024-09-14T04:30:04.295004+0000","last_active":"2024-09-14T04:30:10.299835+0000","last_peered":"2024-09-14T04:30:10.299835+0000","last_clean":"2024-09-14T04:30:10.299835+0000","last_became_active":"2024-09-14T04:30:04.294694+0000","last_became_peered":"2024-09-14T04:30:04.294694+0000","last_unstale":"2024-09-14T04:30:10.299835+0000","last_undegraded":"2024-09-14T04:30:10.299835+0000","last_fullsized":"2024-09-14T04:30:10.299835+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:42.920740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"516'5094","reported_seq":5125,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:35.527440+0000","last_change":"2024-09-14T04:29:19.891388+0000","last_active":"2024-09-14T04:30:35.527440+0000","last_peered":"2024-09-14T04:30:35.527440+0000","last_clean":"2024-09-14T04:30:35.527440+0000","last_became_active":"2024-09-14T04:29:19.891053+0000","last_became_peered":"2024-09-14T04:29:19.891053+0000","last_unstale":"2024-09-14T04:30:35.527440+0000","last_undegraded":"2024-09-14T04:30:35.527440+0000","last_fullsized":"2024-09-14T04:30:35.527440+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:09.607649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333840384,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":326016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":690,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299068+0000","last_change":"2024-09-14T04:29:40.283975+0000","last_active":"2024-09-14T04:30:10.299068+0000","last_peered":"2024-09-14T04:30:10.299068+0000","last_clean":"2024-09-14T04:30:10.299068+0000","last_became_active":"2024-09-14T04:29:40.283613+0000","last_became_peered":"2024-09-14T04:29:40.283613+0000","last_unstale":"2024-09-14T04:30:10.299068+0000","last_undegraded":"2024-09-14T04:30:10.299068+0000","last_fullsized":"2024-09-14T04:30:10.299068+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:57:26.778263+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:30.235232+0000","last_change":"2024-09-14T04:30:04.285245+0000","last_active":"2024-09-14T04:30:30.235232+0000","last_peered":"2024-09-14T04:30:30.235232+0000","last_clean":"2024-09-14T04:30:30.235232+0000","last_became_active":"2024-09-14T04:30:04.284576+0000","last_became_peered":"2024-09-14T04:30:04.284576+0000","last_unstale":"2024-09-14T04:30:30.235232+0000","last_undegraded":"2024-09-14T04:30:30.235232+0000","last_fullsized":"2024-09-14T04:30:30.235232+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:15.875331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":145,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300769+0000","last_change":"2024-09-14T04:30:04.295569+0000","last_active":"2024-09-14T04:30:10.300769+0000","last_peered":"2024-09-14T04:30:10.300769+0000","last_clean":"2024-09-14T04:30:10.300769+0000","last_became_active":"2024-09-14T04:30:04.294571+0000","last_became_peered":"2024-09-14T04:30:04.294571+0000","last_unstale":"2024-09-14T04:30:10.300769+0000","last_undegraded":"2024-09-14T04:30:10.300769+0000","last_fullsized":"2024-09-14T04:30:10.300769+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:40.129021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"516'4895","reported_seq":4926,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:35.221837+0000","last_change":"2024-09-14T04:29:19.670145+0000","last_active":"2024-09-14T04:30:35.221837+0000","last_peered":"2024-09-14T04:30:35.221837+0000","last_clean":"2024-09-14T04:30:35.221837+0000","last_became_active":"2024-09-14T04:29:19.669822+0000","last_became_peered":"2024-09-14T04:29:19.669822+0000","last_unstale":"2024-09-14T04:30:35.221837+0000","last_undegraded":"2024-09-14T04:30:35.221837+0000","last_fullsized":"2024-09-14T04:30:35.221837+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:04.950066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320798720,"num_objects":4895,"num_object_clones":0,"num_object_copies":14685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4895,"num_write_kb":313280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":872,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.679712+0000","last_change":"2024-09-14T04:30:07.295122+0000","last_active":"2024-09-14T04:30:32.679712+0000","last_peered":"2024-09-14T04:30:32.679712+0000","last_clean":"2024-09-14T04:30:32.679712+0000","last_became_active":"2024-09-14T04:30:07.294594+0000","last_became_peered":"2024-09-14T04:30:07.294594+0000","last_unstale":"2024-09-14T04:30:32.679712+0000","last_undegraded":"2024-09-14T04:30:32.679712+0000","last_fullsized":"2024-09-14T04:30:32.679712+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:16.219643+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299469+0000","last_change":"2024-09-14T04:30:04.291481+0000","last_active":"2024-09-14T04:30:10.299469+0000","last_peered":"2024-09-14T04:30:10.299469+0000","last_clean":"2024-09-14T04:30:10.299469+0000","last_became_active":"2024-09-14T04:30:04.291034+0000","last_became_peered":"2024-09-14T04:30:04.291034+0000","last_unstale":"2024-09-14T04:30:10.299469+0000","last_undegraded":"2024-09-14T04:30:10.299469+0000","last_fullsized":"2024-09-14T04:30:10.299469+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:09.399818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":543,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302316+0000","last_change":"2024-09-14T04:30:04.295895+0000","last_active":"2024-09-14T04:30:10.302316+0000","last_peered":"2024-09-14T04:30:10.302316+0000","last_clean":"2024-09-14T04:30:10.302316+0000","last_became_active":"2024-09-14T04:30:04.295549+0000","last_became_peered":"2024-09-14T04:30:04.295549+0000","last_unstale":"2024-09-14T04:30:10.302316+0000","last_undegraded":"2024-09-14T04:30:10.302316+0000","last_fullsized":"2024-09-14T04:30:10.302316+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:22:22.983765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"516'5088","reported_seq":5143,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:34.689082+0000","last_change":"2024-09-14T04:29:19.666943+0000","last_active":"2024-09-14T04:30:34.689082+0000","last_peered":"2024-09-14T04:30:34.689082+0000","last_clean":"2024-09-14T04:30:34.689082+0000","last_became_active":"2024-09-14T04:29:19.666425+0000","last_became_peered":"2024-09-14T04:29:19.666425+0000","last_unstale":"2024-09-14T04:30:34.689082+0000","last_undegraded":"2024-09-14T04:30:34.689082+0000","last_fullsized":"2024-09-14T04:30:34.689082+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:48.691980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333447168,"num_objects":5088,"num_object_clones":0,"num_object_copies":15264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5088,"num_write_kb":325632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":704,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.309833+0000","last_change":"2024-09-14T04:29:40.277117+0000","last_active":"2024-09-14T04:30:10.309833+0000","last_peered":"2024-09-14T04:30:10.309833+0000","last_clean":"2024-09-14T04:30:10.309833+0000","last_became_active":"2024-09-14T04:29:40.276702+0000","last_became_peered":"2024-09-14T04:29:40.276702+0000","last_unstale":"2024-09-14T04:30:10.309833+0000","last_undegraded":"2024-09-14T04:30:10.309833+0000","last_fullsized":"2024-09-14T04:30:10.309833+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:19.375908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:19.375908+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:20.933166+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":232,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.673564+0000","last_change":"2024-09-14T04:30:04.298649+0000","last_active":"2024-09-14T04:30:32.673564+0000","last_peered":"2024-09-14T04:30:32.673564+0000","last_clean":"2024-09-14T04:30:32.673564+0000","last_became_active":"2024-09-14T04:30:04.295253+0000","last_became_peered":"2024-09-14T04:30:04.295253+0000","last_unstale":"2024-09-14T04:30:32.673564+0000","last_undegraded":"2024-09-14T04:30:32.673564+0000","last_fullsized":"2024-09-14T04:30:32.673564+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:23.430095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":155,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300238+0000","last_change":"2024-09-14T04:30:04.296998+0000","last_active":"2024-09-14T04:30:10.300238+0000","last_peered":"2024-09-14T04:30:10.300238+0000","last_clean":"2024-09-14T04:30:10.300238+0000","last_became_active":"2024-09-14T04:30:04.296685+0000","last_became_peered":"2024-09-14T04:30:04.296685+0000","last_unstale":"2024-09-14T04:30:10.300238+0000","last_undegraded":"2024-09-14T04:30:10.300238+0000","last_fullsized":"2024-09-14T04:30:10.300238+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:31.564351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.f","version":"516'4946","reported_seq":4984,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:35.149924+0000","last_change":"2024-09-14T04:30:05.452937+0000","last_active":"2024-09-14T04:30:35.149924+0000","last_peered":"2024-09-14T04:30:35.149924+0000","last_clean":"2024-09-14T04:30:03.643580+0000","last_became_active":"2024-09-14T04:30:05.442092+0000","last_became_peered":"2024-09-14T04:30:05.442092+0000","last_unstale":"2024-09-14T04:30:35.149924+0000","last_undegraded":"2024-09-14T04:30:05.293722+0000","last_fullsized":"2024-09-14T04:30:05.274384+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:30.233993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324141056,"num_objects":4946,"num_object_clones":0,"num_object_copies":14838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3342,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4946,"num_write_kb":316544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":3342},{"shards":"3,7","objects":1604}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":924,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.298971+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:30:10.298971+0000","last_peered":"2024-09-14T04:30:10.298971+0000","last_clean":"2024-09-14T04:30:10.298971+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:30:10.298971+0000","last_undegraded":"2024-09-14T04:30:10.298971+0000","last_fullsized":"2024-09-14T04:30:10.298971+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":416,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300958+0000","last_change":"2024-09-14T04:29:38.001028+0000","last_active":"2024-09-14T04:30:10.300958+0000","last_peered":"2024-09-14T04:30:10.300958+0000","last_clean":"2024-09-14T04:30:10.300958+0000","last_became_active":"2024-09-14T04:29:37.987180+0000","last_became_peered":"2024-09-14T04:29:37.987180+0000","last_unstale":"2024-09-14T04:30:10.300958+0000","last_undegraded":"2024-09-14T04:30:10.300958+0000","last_fullsized":"2024-09-14T04:30:10.300958+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:29.081173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":116,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.309987+0000","last_change":"2024-09-14T04:29:37.992219+0000","last_active":"2024-09-14T04:30:10.309987+0000","last_peered":"2024-09-14T04:30:10.309987+0000","last_clean":"2024-09-14T04:30:10.309987+0000","last_became_active":"2024-09-14T04:29:37.991324+0000","last_became_peered":"2024-09-14T04:29:37.991324+0000","last_unstale":"2024-09-14T04:30:10.309987+0000","last_undegraded":"2024-09-14T04:30:10.309987+0000","last_fullsized":"2024-09-14T04:30:10.309987+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:08.893407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300697+0000","last_change":"2024-09-14T04:27:20.247824+0000","last_active":"2024-09-14T04:30:10.300697+0000","last_peered":"2024-09-14T04:30:10.300697+0000","last_clean":"2024-09-14T04:30:10.300697+0000","last_became_active":"2024-09-14T04:27:20.247205+0000","last_became_peered":"2024-09-14T04:27:20.247205+0000","last_unstale":"2024-09-14T04:30:10.300697+0000","last_undegraded":"2024-09-14T04:30:10.300697+0000","last_fullsized":"2024-09-14T04:30:10.300697+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:09.902975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.c","version":"516'4879","reported_seq":5323,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.707464+0000","last_change":"2024-09-14T04:30:04.292598+0000","last_active":"2024-09-14T04:30:34.707464+0000","last_peered":"2024-09-14T04:30:34.707464+0000","last_clean":"2024-09-14T04:29:36.982014+0000","last_became_active":"2024-09-14T04:29:39.053628+0000","last_became_peered":"2024-09-14T04:29:39.053628+0000","last_unstale":"2024-09-14T04:30:34.707464+0000","last_undegraded":"2024-09-14T04:30:04.291789+0000","last_fullsized":"2024-09-14T04:29:38.987365+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4879,"log_dups_size":0,"ondisk_log_size":4879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319750144,"num_objects":4879,"num_object_clones":0,"num_object_copies":14637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4879,"num_write_kb":312256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":12517376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":770},{"shards":"0,4","objects":4109}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":386,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299306+0000","last_change":"2024-09-14T04:30:07.307496+0000","last_active":"2024-09-14T04:30:10.299306+0000","last_peered":"2024-09-14T04:30:10.299306+0000","last_clean":"2024-09-14T04:30:10.299306+0000","last_became_active":"2024-09-14T04:30:07.304428+0000","last_became_peered":"2024-09-14T04:30:07.304428+0000","last_unstale":"2024-09-14T04:30:10.299306+0000","last_undegraded":"2024-09-14T04:30:10.299306+0000","last_fullsized":"2024-09-14T04:30:10.299306+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":95,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.298802+0000","last_change":"2024-09-14T04:30:04.283624+0000","last_active":"2024-09-14T04:30:10.298802+0000","last_peered":"2024-09-14T04:30:10.298802+0000","last_clean":"2024-09-14T04:30:10.298802+0000","last_became_active":"2024-09-14T04:30:04.283298+0000","last_became_peered":"2024-09-14T04:30:04.283298+0000","last_unstale":"2024-09-14T04:30:10.298802+0000","last_undegraded":"2024-09-14T04:30:10.298802+0000","last_fullsized":"2024-09-14T04:30:10.298802+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:32.022169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":882,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.677244+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:30:32.677244+0000","last_peered":"2024-09-14T04:30:32.677244+0000","last_clean":"2024-09-14T04:30:32.677244+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:30:32.677244+0000","last_undegraded":"2024-09-14T04:30:32.677244+0000","last_fullsized":"2024-09-14T04:30:32.677244+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":658,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.683668+0000","last_change":"2024-09-14T04:30:04.285113+0000","last_active":"2024-09-14T04:30:32.683668+0000","last_peered":"2024-09-14T04:30:32.683668+0000","last_clean":"2024-09-14T04:30:32.683668+0000","last_became_active":"2024-09-14T04:30:04.283783+0000","last_became_peered":"2024-09-14T04:30:04.283783+0000","last_unstale":"2024-09-14T04:30:32.683668+0000","last_undegraded":"2024-09-14T04:30:32.683668+0000","last_fullsized":"2024-09-14T04:30:32.683668+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:13.747229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.d","version":"516'5009","reported_seq":7021,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:35.603896+0000","last_change":"2024-09-14T04:30:10.306945+0000","last_active":"2024-09-14T04:30:35.603896+0000","last_peered":"2024-09-14T04:30:35.603896+0000","last_clean":"2024-09-14T04:30:35.603896+0000","last_became_active":"2024-09-14T04:30:10.303530+0000","last_became_peered":"2024-09-14T04:30:10.303530+0000","last_unstale":"2024-09-14T04:30:35.603896+0000","last_undegraded":"2024-09-14T04:30:35.603896+0000","last_fullsized":"2024-09-14T04:30:35.603896+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5009,"log_dups_size":0,"ondisk_log_size":5009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:45.258629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328269824,"num_objects":5009,"num_object_clones":0,"num_object_copies":15027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5009,"num_write_kb":320576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":980,"num_bytes_recovered":64225280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":782,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299915+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:30:10.299915+0000","last_peered":"2024-09-14T04:30:10.299915+0000","last_clean":"2024-09-14T04:30:10.299915+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:30:10.299915+0000","last_undegraded":"2024-09-14T04:30:10.299915+0000","last_fullsized":"2024-09-14T04:30:10.299915+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":451,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678445+0000","last_change":"2024-09-14T04:30:04.285510+0000","last_active":"2024-09-14T04:30:32.678445+0000","last_peered":"2024-09-14T04:30:32.678445+0000","last_clean":"2024-09-14T04:30:32.678445+0000","last_became_active":"2024-09-14T04:30:04.285170+0000","last_became_peered":"2024-09-14T04:30:04.285170+0000","last_unstale":"2024-09-14T04:30:32.678445+0000","last_undegraded":"2024-09-14T04:30:32.678445+0000","last_fullsized":"2024-09-14T04:30:32.678445+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:32.780039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":870,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678106+0000","last_change":"2024-09-14T04:30:04.283294+0000","last_active":"2024-09-14T04:30:32.678106+0000","last_peered":"2024-09-14T04:30:32.678106+0000","last_clean":"2024-09-14T04:30:32.678106+0000","last_became_active":"2024-09-14T04:30:04.282980+0000","last_became_peered":"2024-09-14T04:30:04.282980+0000","last_unstale":"2024-09-14T04:30:32.678106+0000","last_undegraded":"2024-09-14T04:30:32.678106+0000","last_fullsized":"2024-09-14T04:30:32.678106+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:56.579446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":410,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.680789+0000","last_change":"2024-09-14T04:30:04.287957+0000","last_active":"2024-09-14T04:30:32.680789+0000","last_peered":"2024-09-14T04:30:32.680789+0000","last_clean":"2024-09-14T04:30:32.680789+0000","last_became_active":"2024-09-14T04:30:04.287313+0000","last_became_peered":"2024-09-14T04:30:04.287313+0000","last_unstale":"2024-09-14T04:30:32.680789+0000","last_undegraded":"2024-09-14T04:30:32.680789+0000","last_fullsized":"2024-09-14T04:30:32.680789+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:25.063506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.e","version":"516'5069","reported_seq":7164,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:35.583229+0000","last_change":"2024-09-14T04:30:05.512239+0000","last_active":"2024-09-14T04:30:35.583229+0000","last_peered":"2024-09-14T04:30:35.583229+0000","last_clean":"2024-09-14T04:29:36.982904+0000","last_became_active":"2024-09-14T04:30:05.493974+0000","last_became_peered":"2024-09-14T04:30:05.493974+0000","last_unstale":"2024-09-14T04:30:35.583229+0000","last_undegraded":"2024-09-14T04:30:05.306061+0000","last_fullsized":"2024-09-14T04:30:05.278549+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:50:40.723139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332201984,"num_objects":5069,"num_object_clones":0,"num_object_copies":15207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3393,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5069,"num_write_kb":324416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1023,"num_bytes_recovered":67043328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6],"avail_no_missing":["4","1","2","6"],"object_location_counts":[{"shards":"1,2,4,6","objects":1023},{"shards":"1,4,6","objects":2370},{"shards":"4,6","objects":1676}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":629,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.679370+0000","last_change":"2024-09-14T04:29:40.285341+0000","last_active":"2024-09-14T04:30:32.679370+0000","last_peered":"2024-09-14T04:30:32.679370+0000","last_clean":"2024-09-14T04:30:32.679370+0000","last_became_active":"2024-09-14T04:29:40.284955+0000","last_became_peered":"2024-09-14T04:29:40.284955+0000","last_unstale":"2024-09-14T04:30:32.679370+0000","last_undegraded":"2024-09-14T04:30:32.679370+0000","last_fullsized":"2024-09-14T04:30:32.679370+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:18:51.087521+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":260,"reported_epoch":516,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-14T04:30:32.678772+0000","last_change":"2024-09-14T04:30:04.281103+0000","last_active":"2024-09-14T04:30:32.678772+0000","last_peered":"2024-09-14T04:30:32.678772+0000","last_clean":"2024-09-14T04:14:41.964031+0000","last_became_active":"2024-09-14T04:30:04.280860+0000","last_became_peered":"2024-09-14T04:30:04.280860+0000","last_unstale":"2024-09-14T04:30:32.678772+0000","last_undegraded":"2024-09-14T04:30:04.271129+0000","last_fullsized":"2024-09-14T04:30:32.678772+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":546,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.688967+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:30:32.688967+0000","last_peered":"2024-09-14T04:30:32.688967+0000","last_clean":"2024-09-14T04:30:32.688967+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:30:32.688967+0000","last_undegraded":"2024-09-14T04:30:32.688967+0000","last_fullsized":"2024-09-14T04:30:32.688967+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":36,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300595+0000","last_change":"2024-09-14T04:30:04.281739+0000","last_active":"2024-09-14T04:30:10.300595+0000","last_peered":"2024-09-14T04:30:10.300595+0000","last_clean":"2024-09-14T04:30:10.300595+0000","last_became_active":"2024-09-14T04:30:04.280769+0000","last_became_peered":"2024-09-14T04:30:04.280769+0000","last_unstale":"2024-09-14T04:30:10.300595+0000","last_undegraded":"2024-09-14T04:30:10.300595+0000","last_fullsized":"2024-09-14T04:30:10.300595+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:28.993968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.8","version":"516'5020","reported_seq":5088,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.688283+0000","last_change":"2024-09-14T04:30:05.487515+0000","last_active":"2024-09-14T04:30:34.688283+0000","last_peered":"2024-09-14T04:30:34.688283+0000","last_clean":"2024-09-14T04:29:36.976413+0000","last_became_active":"2024-09-14T04:30:05.481210+0000","last_became_peered":"2024-09-14T04:30:05.481210+0000","last_unstale":"2024-09-14T04:30:34.688283+0000","last_undegraded":"2024-09-14T04:30:05.408080+0000","last_fullsized":"2024-09-14T04:30:05.271956+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5020,"log_dups_size":0,"ondisk_log_size":5020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328990720,"num_objects":5020,"num_object_clones":0,"num_object_copies":15060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4413,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5020,"num_write_kb":321280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1022},{"shards":"0,1,3","objects":2369},{"shards":"0,3","objects":1629}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":880,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.675284+0000","last_change":"2024-09-14T04:27:23.264670+0000","last_active":"2024-09-14T04:30:32.675284+0000","last_peered":"2024-09-14T04:30:32.675284+0000","last_clean":"2024-09-14T04:30:32.675284+0000","last_became_active":"2024-09-14T04:27:23.264241+0000","last_became_peered":"2024-09-14T04:27:23.264241+0000","last_unstale":"2024-09-14T04:30:32.675284+0000","last_undegraded":"2024-09-14T04:30:32.675284+0000","last_fullsized":"2024-09-14T04:30:32.675284+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:27.406676+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":499,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.674941+0000","last_change":"2024-09-14T04:29:37.992555+0000","last_active":"2024-09-14T04:30:32.674941+0000","last_peered":"2024-09-14T04:30:32.674941+0000","last_clean":"2024-09-14T04:30:32.674941+0000","last_became_active":"2024-09-14T04:29:37.992147+0000","last_became_peered":"2024-09-14T04:29:37.992147+0000","last_unstale":"2024-09-14T04:30:32.674941+0000","last_undegraded":"2024-09-14T04:30:32.674941+0000","last_fullsized":"2024-09-14T04:30:32.674941+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:26:44.313121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":811,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.680472+0000","last_change":"2024-09-14T04:30:04.287689+0000","last_active":"2024-09-14T04:30:32.680472+0000","last_peered":"2024-09-14T04:30:32.680472+0000","last_clean":"2024-09-14T04:30:32.680472+0000","last_became_active":"2024-09-14T04:30:04.287280+0000","last_became_peered":"2024-09-14T04:30:04.287280+0000","last_unstale":"2024-09-14T04:30:32.680472+0000","last_undegraded":"2024-09-14T04:30:32.680472+0000","last_fullsized":"2024-09-14T04:30:32.680472+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:19.006536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":153,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299406+0000","last_change":"2024-09-14T04:30:04.294612+0000","last_active":"2024-09-14T04:30:10.299406+0000","last_peered":"2024-09-14T04:30:10.299406+0000","last_clean":"2024-09-14T04:30:10.299406+0000","last_became_active":"2024-09-14T04:30:04.294288+0000","last_became_peered":"2024-09-14T04:30:04.294288+0000","last_unstale":"2024-09-14T04:30:10.299406+0000","last_undegraded":"2024-09-14T04:30:10.299406+0000","last_fullsized":"2024-09-14T04:30:10.299406+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:58.493527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.9","version":"516'5096","reported_seq":5151,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:36.937785+0000","last_change":"2024-09-14T04:29:19.892006+0000","last_active":"2024-09-14T04:30:36.937785+0000","last_peered":"2024-09-14T04:30:36.937785+0000","last_clean":"2024-09-14T04:30:36.937785+0000","last_became_active":"2024-09-14T04:29:19.891032+0000","last_became_peered":"2024-09-14T04:29:19.891032+0000","last_unstale":"2024-09-14T04:30:36.937785+0000","last_undegraded":"2024-09-14T04:30:36.937785+0000","last_fullsized":"2024-09-14T04:30:36.937785+0000","mapping_epoch":494,"log_start":"498'300","ondisk_log_start":"498'300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":4796,"log_dups_size":0,"ondisk_log_size":4796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:43.461118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5096,"num_write_kb":326144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":840,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.673301+0000","last_change":"2024-09-14T04:28:22.699361+0000","last_active":"2024-09-14T04:30:32.673301+0000","last_peered":"2024-09-14T04:30:32.673301+0000","last_clean":"2024-09-14T04:30:32.673301+0000","last_became_active":"2024-09-14T04:28:22.699020+0000","last_became_peered":"2024-09-14T04:28:22.699020+0000","last_unstale":"2024-09-14T04:30:32.673301+0000","last_undegraded":"2024-09-14T04:30:32.673301+0000","last_fullsized":"2024-09-14T04:30:32.673301+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:07.313428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":874,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678488+0000","last_change":"2024-09-14T04:30:04.279568+0000","last_active":"2024-09-14T04:30:32.678488+0000","last_peered":"2024-09-14T04:30:32.678488+0000","last_clean":"2024-09-14T04:30:32.678488+0000","last_became_active":"2024-09-14T04:30:04.279227+0000","last_became_peered":"2024-09-14T04:30:04.279227+0000","last_unstale":"2024-09-14T04:30:32.678488+0000","last_undegraded":"2024-09-14T04:30:32.678488+0000","last_fullsized":"2024-09-14T04:30:32.678488+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:46.120838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":86,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302736+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:30:10.302736+0000","last_peered":"2024-09-14T04:30:10.302736+0000","last_clean":"2024-09-14T04:30:10.302736+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:30:10.302736+0000","last_undegraded":"2024-09-14T04:30:10.302736+0000","last_fullsized":"2024-09-14T04:30:10.302736+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":656,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.688889+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:30:32.688889+0000","last_peered":"2024-09-14T04:30:32.688889+0000","last_clean":"2024-09-14T04:30:32.688889+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:30:32.688889+0000","last_undegraded":"2024-09-14T04:30:32.688889+0000","last_fullsized":"2024-09-14T04:30:32.688889+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"516'5153","reported_seq":5211,"reported_epoch":516,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-14T04:30:38.667875+0000","last_change":"2024-09-14T04:30:05.497361+0000","last_active":"2024-09-14T04:30:38.667875+0000","last_peered":"2024-09-14T04:30:38.667875+0000","last_clean":"2024-09-14T04:30:03.659322+0000","last_became_active":"2024-09-14T04:30:05.487959+0000","last_became_peered":"2024-09-14T04:30:05.487959+0000","last_unstale":"2024-09-14T04:30:38.667875+0000","last_undegraded":"2024-09-14T04:30:38.667875+0000","last_fullsized":"2024-09-14T04:30:38.667875+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5153,"log_dups_size":0,"ondisk_log_size":5153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337707008,"num_objects":5153,"num_object_clones":0,"num_object_copies":15459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3354,"num_objects_unfound":0,"num_objects_dirty":5153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5153,"num_write_kb":329792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[1,5,3],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1799}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":840,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300435+0000","last_change":"2024-09-14T04:29:40.278492+0000","last_active":"2024-09-14T04:30:10.300435+0000","last_peered":"2024-09-14T04:30:10.300435+0000","last_clean":"2024-09-14T04:30:10.300435+0000","last_became_active":"2024-09-14T04:29:40.278037+0000","last_became_peered":"2024-09-14T04:29:40.278037+0000","last_unstale":"2024-09-14T04:30:10.300435+0000","last_undegraded":"2024-09-14T04:30:10.300435+0000","last_fullsized":"2024-09-14T04:30:10.300435+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:11.508248+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":90,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.301333+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:30:10.301333+0000","last_peered":"2024-09-14T04:30:10.301333+0000","last_clean":"2024-09-14T04:30:10.301333+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:30:10.301333+0000","last_undegraded":"2024-09-14T04:30:10.301333+0000","last_fullsized":"2024-09-14T04:30:10.301333+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":774,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.310377+0000","last_change":"2024-09-14T04:30:04.291109+0000","last_active":"2024-09-14T04:30:10.310377+0000","last_peered":"2024-09-14T04:30:10.310377+0000","last_clean":"2024-09-14T04:30:10.310377+0000","last_became_active":"2024-09-14T04:30:04.290804+0000","last_became_peered":"2024-09-14T04:30:04.290804+0000","last_unstale":"2024-09-14T04:30:10.310377+0000","last_undegraded":"2024-09-14T04:30:10.310377+0000","last_fullsized":"2024-09-14T04:30:10.310377+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:05.742866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299137+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:30:10.299137+0000","last_peered":"2024-09-14T04:30:10.299137+0000","last_clean":"2024-09-14T04:30:10.299137+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:30:10.299137+0000","last_undegraded":"2024-09-14T04:30:10.299137+0000","last_fullsized":"2024-09-14T04:30:10.299137+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"516'5034","reported_seq":7010,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:30:34.665739+0000","last_change":"2024-09-14T04:30:05.533783+0000","last_active":"2024-09-14T04:30:34.665739+0000","last_peered":"2024-09-14T04:30:34.665739+0000","last_clean":"2024-09-14T04:30:03.664717+0000","last_became_active":"2024-09-14T04:30:05.522117+0000","last_became_peered":"2024-09-14T04:30:05.522117+0000","last_unstale":"2024-09-14T04:30:34.665739+0000","last_undegraded":"2024-09-14T04:30:05.394334+0000","last_fullsized":"2024-09-14T04:30:05.275171+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:29:18.656566+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.656566+0000","objects_scrubbed":0,"log_size":5034,"log_dups_size":0,"ondisk_log_size":5034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329908224,"num_objects":5034,"num_object_clones":0,"num_object_copies":15102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5034,"num_write_kb":322176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":954,"num_bytes_recovered":62521344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7],"avail_no_missing":["0","1","2","7"],"object_location_counts":[{"shards":"0,1,7","objects":3371},{"shards":"0,7","objects":1663}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1169,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.675822+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:30:32.675822+0000","last_peered":"2024-09-14T04:30:32.675822+0000","last_clean":"2024-09-14T04:30:32.675822+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:30:32.675822+0000","last_undegraded":"2024-09-14T04:30:32.675822+0000","last_fullsized":"2024-09-14T04:30:32.675822+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":659,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.683457+0000","last_change":"2024-09-14T04:30:04.305324+0000","last_active":"2024-09-14T04:30:32.683457+0000","last_peered":"2024-09-14T04:30:32.683457+0000","last_clean":"2024-09-14T04:30:32.683457+0000","last_became_active":"2024-09-14T04:30:04.289343+0000","last_became_peered":"2024-09-14T04:30:04.289343+0000","last_unstale":"2024-09-14T04:30:32.683457+0000","last_undegraded":"2024-09-14T04:30:32.683457+0000","last_fullsized":"2024-09-14T04:30:32.683457+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:41.011672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":76,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:30.235280+0000","last_change":"2024-09-14T04:29:37.991840+0000","last_active":"2024-09-14T04:30:30.235280+0000","last_peered":"2024-09-14T04:30:30.235280+0000","last_clean":"2024-09-14T04:30:30.235280+0000","last_became_active":"2024-09-14T04:29:37.991503+0000","last_became_peered":"2024-09-14T04:29:37.991503+0000","last_unstale":"2024-09-14T04:30:30.235280+0000","last_undegraded":"2024-09-14T04:30:30.235280+0000","last_fullsized":"2024-09-14T04:30:30.235280+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:22.858456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":512,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:30.560400+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:30:30.560400+0000","last_peered":"2024-09-14T04:30:30.560400+0000","last_clean":"2024-09-14T04:30:30.560400+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:30:30.560400+0000","last_undegraded":"2024-09-14T04:30:30.560400+0000","last_fullsized":"2024-09-14T04:30:30.560400+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":566,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299455+0000","last_change":"2024-09-14T04:30:04.291353+0000","last_active":"2024-09-14T04:30:10.299455+0000","last_peered":"2024-09-14T04:30:10.299455+0000","last_clean":"2024-09-14T04:30:10.299455+0000","last_became_active":"2024-09-14T04:30:04.290907+0000","last_became_peered":"2024-09-14T04:30:04.290907+0000","last_unstale":"2024-09-14T04:30:10.299455+0000","last_undegraded":"2024-09-14T04:30:10.299455+0000","last_fullsized":"2024-09-14T04:30:10.299455+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:17.989312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":576,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300010+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:30:10.300010+0000","last_peered":"2024-09-14T04:30:10.300010+0000","last_clean":"2024-09-14T04:30:10.300010+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:30:10.300010+0000","last_undegraded":"2024-09-14T04:30:10.300010+0000","last_fullsized":"2024-09-14T04:30:10.300010+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":445,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.679132+0000","last_change":"2024-09-14T04:30:04.289913+0000","last_active":"2024-09-14T04:30:32.679132+0000","last_peered":"2024-09-14T04:30:32.679132+0000","last_clean":"2024-09-14T04:30:32.679132+0000","last_became_active":"2024-09-14T04:30:04.288970+0000","last_became_peered":"2024-09-14T04:30:04.288970+0000","last_unstale":"2024-09-14T04:30:32.679132+0000","last_undegraded":"2024-09-14T04:30:32.679132+0000","last_fullsized":"2024-09-14T04:30:32.679132+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:20.483087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":92,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302684+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:30:10.302684+0000","last_peered":"2024-09-14T04:30:10.302684+0000","last_clean":"2024-09-14T04:30:10.302684+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:30:10.302684+0000","last_undegraded":"2024-09-14T04:30:10.302684+0000","last_fullsized":"2024-09-14T04:30:10.302684+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":996,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300681+0000","last_change":"2024-09-14T04:30:07.293131+0000","last_active":"2024-09-14T04:30:10.300681+0000","last_peered":"2024-09-14T04:30:10.300681+0000","last_clean":"2024-09-14T04:30:10.300681+0000","last_became_active":"2024-09-14T04:30:07.292809+0000","last_became_peered":"2024-09-14T04:30:07.292809+0000","last_unstale":"2024-09-14T04:30:10.300681+0000","last_undegraded":"2024-09-14T04:30:10.300681+0000","last_fullsized":"2024-09-14T04:30:10.300681+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:15.733204+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":377,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300262+0000","last_change":"2024-09-14T04:30:04.283436+0000","last_active":"2024-09-14T04:30:10.300262+0000","last_peered":"2024-09-14T04:30:10.300262+0000","last_clean":"2024-09-14T04:30:10.300262+0000","last_became_active":"2024-09-14T04:30:04.283122+0000","last_became_peered":"2024-09-14T04:30:04.283122+0000","last_unstale":"2024-09-14T04:30:10.300262+0000","last_undegraded":"2024-09-14T04:30:10.300262+0000","last_fullsized":"2024-09-14T04:30:10.300262+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.646724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":95,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299321+0000","last_change":"2024-09-14T04:28:19.257505+0000","last_active":"2024-09-14T04:30:10.299321+0000","last_peered":"2024-09-14T04:30:10.299321+0000","last_clean":"2024-09-14T04:30:10.299321+0000","last_became_active":"2024-09-14T04:28:19.257193+0000","last_became_peered":"2024-09-14T04:28:19.257193+0000","last_unstale":"2024-09-14T04:30:10.299321+0000","last_undegraded":"2024-09-14T04:30:10.299321+0000","last_fullsized":"2024-09-14T04:30:10.299321+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:03:33.146736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":643,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.681049+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:30:32.681049+0000","last_peered":"2024-09-14T04:30:32.681049+0000","last_clean":"2024-09-14T04:30:32.681049+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:30:32.681049+0000","last_undegraded":"2024-09-14T04:30:32.681049+0000","last_fullsized":"2024-09-14T04:30:32.681049+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":754,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.673853+0000","last_change":"2024-09-14T04:30:04.298334+0000","last_active":"2024-09-14T04:30:32.673853+0000","last_peered":"2024-09-14T04:30:32.673853+0000","last_clean":"2024-09-14T04:30:32.673853+0000","last_became_active":"2024-09-14T04:30:04.294584+0000","last_became_peered":"2024-09-14T04:30:04.294584+0000","last_unstale":"2024-09-14T04:30:32.673853+0000","last_undegraded":"2024-09-14T04:30:32.673853+0000","last_fullsized":"2024-09-14T04:30:32.673853+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:02.616788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":76,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:25.234336+0000","last_change":"2024-09-14T04:30:04.283086+0000","last_active":"2024-09-14T04:30:25.234336+0000","last_peered":"2024-09-14T04:30:25.234336+0000","last_clean":"2024-09-14T04:30:25.234336+0000","last_became_active":"2024-09-14T04:30:04.282704+0000","last_became_peered":"2024-09-14T04:30:04.282704+0000","last_unstale":"2024-09-14T04:30:25.234336+0000","last_undegraded":"2024-09-14T04:30:25.234336+0000","last_fullsized":"2024-09-14T04:30:25.234336+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:03.488767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":17,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.298887+0000","last_change":"2024-09-14T04:30:04.283463+0000","last_active":"2024-09-14T04:30:10.298887+0000","last_peered":"2024-09-14T04:30:10.298887+0000","last_clean":"2024-09-14T04:30:10.298887+0000","last_became_active":"2024-09-14T04:30:04.283157+0000","last_became_peered":"2024-09-14T04:30:04.283157+0000","last_unstale":"2024-09-14T04:30:10.298887+0000","last_undegraded":"2024-09-14T04:30:10.298887+0000","last_fullsized":"2024-09-14T04:30:10.298887+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:02:11.942507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":666,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.683349+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:30:32.683349+0000","last_peered":"2024-09-14T04:30:32.683349+0000","last_clean":"2024-09-14T04:30:32.683349+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:30:32.683349+0000","last_undegraded":"2024-09-14T04:30:32.683349+0000","last_fullsized":"2024-09-14T04:30:32.683349+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":692,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678003+0000","last_change":"2024-09-14T04:30:04.298473+0000","last_active":"2024-09-14T04:30:32.678003+0000","last_peered":"2024-09-14T04:30:32.678003+0000","last_clean":"2024-09-14T04:30:32.678003+0000","last_became_active":"2024-09-14T04:30:04.295176+0000","last_became_peered":"2024-09-14T04:30:04.295176+0000","last_unstale":"2024-09-14T04:30:32.678003+0000","last_undegraded":"2024-09-14T04:30:32.678003+0000","last_fullsized":"2024-09-14T04:30:32.678003+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:02:30.762732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":247,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.302188+0000","last_change":"2024-09-14T04:29:18.672386+0000","last_active":"2024-09-14T04:30:10.302188+0000","last_peered":"2024-09-14T04:30:10.302188+0000","last_clean":"2024-09-14T04:30:10.302188+0000","last_became_active":"2024-09-14T04:29:18.672086+0000","last_became_peered":"2024-09-14T04:29:18.672086+0000","last_unstale":"2024-09-14T04:30:10.302188+0000","last_undegraded":"2024-09-14T04:30:10.302188+0000","last_fullsized":"2024-09-14T04:30:10.302188+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:01.909079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":124,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299148+0000","last_change":"2024-09-14T04:30:04.295082+0000","last_active":"2024-09-14T04:30:10.299148+0000","last_peered":"2024-09-14T04:30:10.299148+0000","last_clean":"2024-09-14T04:30:10.299148+0000","last_became_active":"2024-09-14T04:30:04.293846+0000","last_became_peered":"2024-09-14T04:30:04.293846+0000","last_unstale":"2024-09-14T04:30:10.299148+0000","last_undegraded":"2024-09-14T04:30:10.299148+0000","last_fullsized":"2024-09-14T04:30:10.299148+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:05.873391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":282,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678597+0000","last_change":"2024-09-14T04:30:04.293298+0000","last_active":"2024-09-14T04:30:32.678597+0000","last_peered":"2024-09-14T04:30:32.678597+0000","last_clean":"2024-09-14T04:30:32.678597+0000","last_became_active":"2024-09-14T04:30:04.292876+0000","last_became_peered":"2024-09-14T04:30:04.292876+0000","last_unstale":"2024-09-14T04:30:32.678597+0000","last_undegraded":"2024-09-14T04:30:32.678597+0000","last_fullsized":"2024-09-14T04:30:32.678597+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:13.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":670,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.688780+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:30:32.688780+0000","last_peered":"2024-09-14T04:30:32.688780+0000","last_clean":"2024-09-14T04:30:32.688780+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:30:32.688780+0000","last_undegraded":"2024-09-14T04:30:32.688780+0000","last_fullsized":"2024-09-14T04:30:32.688780+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":136,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.490598+0000","last_change":"2024-09-14T04:30:04.296875+0000","last_active":"2024-09-14T04:30:10.490598+0000","last_peered":"2024-09-14T04:30:10.490598+0000","last_clean":"2024-09-14T04:30:10.490598+0000","last_became_active":"2024-09-14T04:30:04.296576+0000","last_became_peered":"2024-09-14T04:30:04.296576+0000","last_unstale":"2024-09-14T04:30:10.490598+0000","last_undegraded":"2024-09-14T04:30:10.490598+0000","last_fullsized":"2024-09-14T04:30:10.490598+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:40.996256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":234,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299687+0000","last_change":"2024-09-14T04:30:04.303745+0000","last_active":"2024-09-14T04:30:10.299687+0000","last_peered":"2024-09-14T04:30:10.299687+0000","last_clean":"2024-09-14T04:30:10.299687+0000","last_became_active":"2024-09-14T04:30:04.291531+0000","last_became_peered":"2024-09-14T04:30:04.291531+0000","last_unstale":"2024-09-14T04:30:10.299687+0000","last_undegraded":"2024-09-14T04:30:10.299687+0000","last_fullsized":"2024-09-14T04:30:10.299687+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:01:41.726419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.490567+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:30:10.490567+0000","last_peered":"2024-09-14T04:30:10.490567+0000","last_clean":"2024-09-14T04:30:10.490567+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:30:10.490567+0000","last_undegraded":"2024-09-14T04:30:10.490567+0000","last_fullsized":"2024-09-14T04:30:10.490567+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":412,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.300356+0000","last_change":"2024-09-14T04:30:04.305524+0000","last_active":"2024-09-14T04:30:10.300356+0000","last_peered":"2024-09-14T04:30:10.300356+0000","last_clean":"2024-09-14T04:30:10.300356+0000","last_became_active":"2024-09-14T04:30:04.295096+0000","last_became_peered":"2024-09-14T04:30:04.295096+0000","last_unstale":"2024-09-14T04:30:10.300356+0000","last_undegraded":"2024-09-14T04:30:10.300356+0000","last_fullsized":"2024-09-14T04:30:10.300356+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:12:08.394662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":391,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.678533+0000","last_change":"2024-09-14T04:29:38.010269+0000","last_active":"2024-09-14T04:30:32.678533+0000","last_peered":"2024-09-14T04:30:32.678533+0000","last_clean":"2024-09-14T04:30:32.678533+0000","last_became_active":"2024-09-14T04:29:38.009875+0000","last_became_peered":"2024-09-14T04:29:38.009875+0000","last_unstale":"2024-09-14T04:30:32.678533+0000","last_undegraded":"2024-09-14T04:30:32.678533+0000","last_fullsized":"2024-09-14T04:30:32.678533+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:38.314967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":104,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299669+0000","last_change":"2024-09-14T04:30:04.281683+0000","last_active":"2024-09-14T04:30:10.299669+0000","last_peered":"2024-09-14T04:30:10.299669+0000","last_clean":"2024-09-14T04:30:10.299669+0000","last_became_active":"2024-09-14T04:30:04.280833+0000","last_became_peered":"2024-09-14T04:30:04.280833+0000","last_unstale":"2024-09-14T04:30:10.299669+0000","last_undegraded":"2024-09-14T04:30:10.299669+0000","last_fullsized":"2024-09-14T04:30:10.299669+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:49.260981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":508,"reported_epoch":515,"state":"active+clean","last_fresh":"2024-09-14T04:30:10.299325+0000","last_change":"2024-09-14T04:30:04.300041+0000","last_active":"2024-09-14T04:30:10.299325+0000","last_peered":"2024-09-14T04:30:10.299325+0000","last_clean":"2024-09-14T04:30:10.299325+0000","last_became_active":"2024-09-14T04:30:04.299711+0000","last_became_peered":"2024-09-14T04:30:04.299711+0000","last_unstale":"2024-09-14T04:30:10.299325+0000","last_undegraded":"2024-09-14T04:30:10.299325+0000","last_fullsized":"2024-09-14T04:30:10.299325+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:30:31.280922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":704,"reported_epoch":516,"state":"active+clean","last_fresh":"2024-09-14T04:30:32.681144+0000","last_change":"2024-09-14T04:29:38.016663+0000","last_active":"2024-09-14T04:30:32.681144+0000","last_peered":"2024-09-14T04:30:32.681144+0000","last_clean":"2024-09-14T04:30:32.681144+0000","last_became_active":"2024-09-14T04:29:38.007007+0000","last_became_peered":"2024-09-14T04:29:38.007007+0000","last_unstale":"2024-09-14T04:30:32.681144+0000","last_undegraded":"2024-09-14T04:30:32.681144+0000","last_fullsized":"2024-09-14T04:30:32.681144+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:55:55.070473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":81,"num_bytes_recovered":22506,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":5261688832,"num_objects":80287,"num_object_clones":0,"num_object_copies":240861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27455,"num_objects_misplaced":5334,"num_objects_unfound":0,"num_objects_dirty":80287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":80287,"num_write_kb":5138368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5652,"num_bytes_recovered":370409472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16220094464,"data_stored":16220094464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79687,"ondisk_log_size":79687,"up":48,"acting":38,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":3070502,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201880,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3037276,"kb_used_data":1597392,"kb_used_omap":1,"kb_used_meta":1439870,"kb_avail":90707876,"statfs":{"total":95995035648,"available":92884865024,"internally_reserved":0,"allocated":1635729408,"data_stored":1632338187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1474427339},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463518,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2944332,"kb_used_data":1898436,"kb_used_omap":1,"kb_used_meta":1045886,"kb_avail":90800820,"statfs":{"total":95995035648,"available":92980039680,"internally_reserved":0,"allocated":1943998464,"data_stored":1940619447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1070987719},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834191,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395976,"kb_used_data":1931328,"kb_used_omap":5,"kb_used_meta":464634,"kb_avail":91349176,"statfs":{"total":95995035648,"available":93541556224,"internally_reserved":0,"allocated":1977679872,"data_stored":1974300997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":475785277},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986790,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3992536,"kb_used_data":2421964,"kb_used_omap":1,"kb_used_meta":1570558,"kb_avail":89752616,"statfs":{"total":95995035648,"available":91906678784,"internally_reserved":0,"allocated":2480091136,"data_stored":2476704212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1608251846},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748819,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3964420,"kb_used_data":2789820,"kb_used_omap":4,"kb_used_meta":1174587,"kb_avail":89780732,"statfs":{"total":95995035648,"available":91935469568,"internally_reserved":0,"allocated":2856775680,"data_stored":2853400613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1202777644},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":1275605287040,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":463856468363,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342477,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3474120,"kb_used_data":2654272,"kb_used_omap":10,"kb_used_meta":819829,"kb_avail":90271032,"statfs":{"total":95995035648,"available":92437536768,"internally_reserved":0,"allocated":2717974528,"data_stored":2714595354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":839505106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2722758656,"data_stored":2722758656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2115633152,"data_stored":2115633152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":427032576,"data_stored":427032576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2869821440,"data_stored":2869821440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2507014144,"data_stored":2507014144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1988296704,"data_stored":1988296704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1950941184,"data_stored":1950941184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1638596608,"data_stored":1638596608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:30:42.122 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:30:42.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_9 pg_num 26 2024-09-14T04:30:42.133 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:42.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:42.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:42.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:42.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:42.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 71 16 85696 85680 75.4125 48.875 0.0127879 0.0132548 2024-09-14T04:30:42.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:42.571 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:42.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:42.703 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:42.805 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:42.840 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:42.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:42.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:42 smithi001 ceph-mon[112165]: pgmap v1630: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 27456/250497 objects degraded (10.961%); 5334/250497 objects misplaced (2.129%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:42.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:42 smithi001 ceph-mon[107399]: pgmap v1630: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 27456/250497 objects degraded (10.961%); 5334/250497 objects misplaced (2.129%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:42.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:42.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:43.077 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:42 smithi012 ceph-mon[101180]: pgmap v1630: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 27456/250497 objects degraded (10.961%); 5334/250497 objects misplaced (2.129%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:30:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.117 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:43.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:43.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:43.258 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:43.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:43.390 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:43.449 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 72 16 86558 86542 75.1133 53.875 0.0187065 0.0133078 2024-09-14T04:30:43.492 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:43.523 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:43.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:43.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:43.789 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:43.821 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:43.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:43.963 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:43.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:43.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:44.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:44.097 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:44.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:44.236 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:44.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:44.384 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:44.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 73 16 87345 87329 74.7581 49.1875 0.0249439 0.0133702 2024-09-14T04:30:44.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:44.531 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:44.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:44.671 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:44.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:44.761 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:44 smithi001 ceph-mon[107399]: pgmap v1631: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 27456/251838 objects degraded (10.902%); 5095/251838 objects misplaced (2.023%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:30:44.761 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/251838 objects degraded (10.902%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:44.762 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:44 smithi001 ceph-mon[112165]: pgmap v1631: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 27456/251838 objects degraded (10.902%); 5095/251838 objects misplaced (2.023%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:30:44.762 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/251838 objects degraded (10.902%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:44.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:44.813 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:44.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:44.984 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:44.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:45.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:45.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:44 smithi012 ceph-mon[101180]: pgmap v1631: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 27456/251838 objects degraded (10.902%); 5095/251838 objects misplaced (2.023%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:30:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/251838 objects degraded (10.902%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:45.122 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:45.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.263 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:45.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:45.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:45.401 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:45.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 74 16 88143 88127 74.4219 49.875 0.0082804 0.0134306 2024-09-14T04:30:45.502 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:45.523 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:30:45.523 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:30:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.547 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:45.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:45.639 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:30:45.639 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:30:45.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:45.650 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:45.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:45.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:45.687 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:30:45.688 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:30:45.741 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:30:45.741 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:30:45.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:45.794 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:30:45.795 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:30:45.832 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:30:45.832 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:30:45.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:45.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1091517155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:45.873 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:30:45.873 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:30:45.910 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:30:45.910 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:30:45.911 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1091517155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:45.911 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1091517155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:30:45.939 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:45.943 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:30:45.943 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:30:45.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:45.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:46.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:46.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:46.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:46.238 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.270 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:46.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:46.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:30:46.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:46.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 75 16 88917 88901 74.0745 48.375 0.00940888 0.0134938 2024-09-14T04:30:46.456 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:46.558 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:46.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:46.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:46.699 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:30:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.744 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:46.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:46.765 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:46.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:46.897 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:46.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:46.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[107399]: pgmap v1632: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 56 MiB/s wr, 889 op/s; 27456/260016 objects degraded (10.559%); 5095/260016 objects misplaced (1.959%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/322687514' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[112165]: pgmap v1632: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 56 MiB/s wr, 889 op/s; 27456/260016 objects degraded (10.559%); 5095/260016 objects misplaced (1.959%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/322687514' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:46] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.51.0" 2024-09-14T04:30:47.001 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:47.034 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:47.055 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:46 smithi012 ceph-mon[101180]: pgmap v1632: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 56 MiB/s wr, 889 op/s; 27456/260016 objects degraded (10.559%); 5095/260016 objects misplaced (1.959%); 2.5 MiB/s, 40 objects/s recovering 2024-09-14T04:30:47.056 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/322687514' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:47.056 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:30:47.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:47.174 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:47.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:47.306 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:47.408 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:47.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:47.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 76 16 89687 89671 73.7331 48.125 0.0086845 0.013557 2024-09-14T04:30:47.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:47.586 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:47.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:47.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:47.825 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:47.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:47.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:47.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:47.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:30:47.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[107399]: osdmap e517: 8 total, 8 up, 6 in 2024-09-14T04:30:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:30:47.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:47 smithi001 ceph-mon[112165]: osdmap e517: 8 total, 8 up, 6 in 2024-09-14T04:30:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:47.990 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:47.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:48.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:47 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:30:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:47 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:30:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:47 smithi012 ceph-mon[101180]: osdmap e517: 8 total, 8 up, 6 in 2024-09-14T04:30:48.131 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:48.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:48.233 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:48.265 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:48.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:48.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:48.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 77 16 90517 90501 73.4491 51.875 0.0287038 0.0136087 2024-09-14T04:30:48.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:48.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:48.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:48.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:48.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:48.864 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:48.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:48.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:48.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:48 smithi001 ceph-mon[107399]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/263868 objects degraded (10.405%); 4835/263868 objects misplaced (1.832%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:48 smithi001 ceph-mon[112165]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/263868 objects degraded (10.405%); 4835/263868 objects misplaced (1.832%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:49.013 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:48 smithi012 ceph-mon[101180]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/263868 objects degraded (10.405%); 4835/263868 objects misplaced (1.832%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:49.017 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:49.106 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:30:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:30:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:30:49.106 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:30:49.107 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:30:49.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:49.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:49.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:49.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:49.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:49.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 78 16 91282 91266 73.1204 47.8125 0.0165797 0.0136692 2024-09-14T04:30:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:49.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:49.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:49.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:49.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:49.696 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/263868 objects degraded (10.405%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:49.730 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:49.832 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:49.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:49.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:49.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:30:49.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:49.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/263868 objects degraded (10.405%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/263868 objects degraded (10.405%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.006 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:50.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:50.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:50.247 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:50.268 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:50.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:50.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 79 16 92045 92029 72.7985 47.6875 0.0198681 0.0137293 2024-09-14T04:30:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:50.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:50.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:50.712 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:50.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:50.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:50.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:50.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:50.933 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:50 smithi012 ceph-mon[101180]: pgmap v1635: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 789 op/s; 27456/264624 objects degraded (10.375%); 4835/264624 objects misplaced (1.827%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:50.934 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:50.934 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:50.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[107399]: pgmap v1635: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 789 op/s; 27456/264624 objects degraded (10.375%); 4835/264624 objects misplaced (1.827%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[112165]: pgmap v1635: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 789 op/s; 27456/264624 objects degraded (10.375%); 4835/264624 objects misplaced (1.827%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:50.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:50.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:51.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:51.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:51.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:51.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:51.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:51.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:51.431 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:51.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:51.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:2024-09-14T04:30:51.450346+0000 min lat: 0.00130141 max lat: 2.15405 avg lat: 0.0137798 2024-09-14T04:30:51.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:30:51.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 80 16 92875 92859 72.5369 51.875 0.0108334 0.0137798 2024-09-14T04:30:51.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:51.572 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:51.674 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:51.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:51.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:51.850 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:51.952 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:51.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:51 smithi001 ceph-mon[107399]: pgmap v1636: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 27456/274131 objects degraded (10.016%); 4835/274131 objects misplaced (1.764%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:51 smithi001 ceph-mon[112165]: pgmap v1636: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 27456/274131 objects degraded (10.016%); 4835/274131 objects misplaced (1.764%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:51.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:51.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:51.992 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:51 smithi012 ceph-mon[101180]: pgmap v1636: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 27456/274131 objects degraded (10.016%); 4835/274131 objects misplaced (1.764%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:30:52.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.124 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:52.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:52.225 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:52.256 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:52.360 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:52.395 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:52.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 81 16 93673 93657 72.257 49.875 0.0203656 0.0138312 2024-09-14T04:30:52.497 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:52.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:52.556 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:30:52.556 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223251 13.f 13:fb213dff:::benchmark_data_smithi001_306460_object105007:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.657466+0000", 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.89509513499999, 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0689271439999999, 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:30:52.557 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223251 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.657466+0000", 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.657466+0000", 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.657464+0000", 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.657471+0000", 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5079999999999999e-06 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.657493+0000", 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1917e-05 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661652+0000", 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041587489999999998 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662693+0000", 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00104159 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662701+0000", 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0010000000000001e-06 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726339+0000", 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.063637551 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726393+0000", 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4163999999999998e-05 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223260 13.f 13:fd611c5c:::benchmark_data_smithi001_306460_object105016:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.661684+0000", 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.890876988, 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0650818399999999, 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223260 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661684+0000", 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661684+0000", 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661683+0000", 2024-09-14T04:30:52.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661689+0000", 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3640000000000004e-06 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661701+0000", 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1926999999999999e-05 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661709+0000", 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6909999999999997e-06 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662723+0000", 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010146560000000001 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662729+0000", 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4489999999999998e-06 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726695+0000", 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.063966593 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726766+0000", 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0685000000000005e-05 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223262 13.f 13:f8c767ff:::benchmark_data_smithi001_306460_object105018:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.661715+0000", 2024-09-14T04:30:52.569 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.890846651, 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0651411900000001, 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223262 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661715+0000", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661715+0000", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661714+0000", 2024-09-14T04:30:52.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661717+0000", 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3239999999999999e-06 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661724+0000", 2024-09-14T04:30:52.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.815e-06 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.661729+0000", 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2179999999999998e-06 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662749+0000", 2024-09-14T04:30:52.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010191810000000001 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.662753+0000", 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8779999999999997e-06 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726805+0000", 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.064051378 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726856+0000", 2024-09-14T04:30:52.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0832999999999999e-05 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223299 13.f 13:f0e8b68f:::benchmark_data_smithi001_306460_object105055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.686400+0000", 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.866160831, 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0664134919999999, 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.576 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223299 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686400+0000", 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686400+0000", 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686398+0000", 2024-09-14T04:30:52.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686407+0000", 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8529999999999998e-06 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686427+0000", 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9559e-05 2024-09-14T04:30:52.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686435+0000", 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0960000000000001e-06 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686490+0000", 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5574e-05 2024-09-14T04:30:52.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.686502+0000", 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1659e-05 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.726917+0000", 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0404153679999999 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.727022+0000", 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104397 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.727141+0000", 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118804 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.736849+0000", 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0097082039999999998 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741253+0000", 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044042029999999998 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752724+0000", 2024-09-14T04:30:52.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011470531000000001 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752775+0000", 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1866000000000001e-05 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752814+0000", 2024-09-14T04:30:52.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8418999999999999e-05 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223387 13.f 13:f6c65659:::benchmark_data_smithi001_306460_object105143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.761323+0000", 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.79123767499999, 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.004831934, 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.585 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223387 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761323+0000", 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761323+0000", 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761322+0000", 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761330+0000", 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3820000000000004e-06 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761345+0000", 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5122e-05 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761355+0000", 2024-09-14T04:30:52.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3079999999999997e-06 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761409+0000", 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4540000000000003e-05 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.761418+0000", 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1719999999999996e-06 2024-09-14T04:30:52.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.729965+0000", 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96854720000000005 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.730071+0000", 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010575 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.730177+0000", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105611 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741259+0000", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011082249000000001 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.743945+0000", 2024-09-14T04:30:52.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026858670000000002 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766063+0000", 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022118021000000002 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766116+0000", 2024-09-14T04:30:52.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3468999999999999e-05 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766155+0000", 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8964000000000002e-05 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.593 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223389 13.f 13:fcbfb53c:::benchmark_data_smithi001_306460_object105145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.763097+0000", 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.78946387799999, 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.003160192, 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223389 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.594 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763097+0000", 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763097+0000", 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763096+0000", 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763105+0000", 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6750000000000008e-06 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763121+0000", 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6049000000000002e-05 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763128+0000", 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0840000000000003e-06 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763170+0000", 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2432999999999998e-05 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.763179+0000", 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6680000000000007e-06 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.732285+0000", 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96910573 2024-09-14T04:30:52.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.732359+0000", 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4371999999999999e-05 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.732425+0000", 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6305999999999998e-05 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741265+0000", 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0088394340000000002 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.744009+0000", 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027437809999999998 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766190+0000", 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022181289 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766226+0000", 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5768e-05 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766257+0000", 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1782000000000002e-05 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223421 13.f 13:fa8bd8cf:::benchmark_data_smithi001_306460_object105177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.791340+0000", 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.76122129199999, 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.975016726, 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.602 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223421 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791340+0000", 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791340+0000", 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791338+0000", 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791345+0000", 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.754e-06 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791361+0000", 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6467999999999999e-05 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791369+0000", 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4739999999999999e-06 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791422+0000", 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2812999999999997e-05 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.791433+0000", 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1735e-05 2024-09-14T04:30:52.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.734462+0000", 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94302883999999998 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.734551+0000", 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8988999999999998e-05 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.734617+0000", 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5395000000000007e-05 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741271+0000", 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066539299999999997 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.744058+0000", 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027873590000000001 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766290+0000", 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022232368999999998 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766325+0000", 2024-09-14T04:30:52.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4387999999999997e-05 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766357+0000", 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1872000000000002e-05 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223441 13.f 13:fd125881:::benchmark_data_smithi001_306460_object105197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.610 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.807302+0000", 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.74525876499999, 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96407051099999996, 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223441 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807302+0000", 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807302+0000", 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807300+0000", 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807308+0000", 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8050000000000007e-06 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807323+0000", 2024-09-14T04:30:52.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5264000000000002e-05 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807329+0000", 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.93e-06 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807377+0000", 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7460999999999998e-05 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.807383+0000", 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8200000000000002e-06 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.736872+0000", 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92948971499999999 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.736988+0000", 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115955 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.737099+0000", 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011027999999999999 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.743952+0000", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068533079999999998 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752887+0000", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089347330000000003 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771285+0000", 2024-09-14T04:30:52.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018398649999999999 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771333+0000", 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7935000000000001e-05 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771373+0000", 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9660999999999999e-05 2024-09-14T04:30:52.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223454 13.f 13:ff0a0e12:::benchmark_data_smithi001_306460_object105210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.820765+0000", 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.73179630499999, 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95070987100000004, 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.619 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223454 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820765+0000", 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820765+0000", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820763+0000", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820771+0000", 2024-09-14T04:30:52.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2800000000000003e-06 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820787+0000", 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5464e-05 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820795+0000", 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1219999999999995e-06 2024-09-14T04:30:52.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820845+0000", 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0494000000000001e-05 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.820853+0000", 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.576e-06 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741300+0000", 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92044697499999995 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741405+0000", 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104786 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.741465+0000", 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0365000000000002e-05 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752827+0000", 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011361598000000001 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766030+0000", 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013202844 2024-09-14T04:30:52.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771407+0000", 2024-09-14T04:30:52.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053776420000000002 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771443+0000", 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5899999999999998e-05 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771475+0000", 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.154e-05 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.627 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223460 13.f 13:f0e34259:::benchmark_data_smithi001_306460_object105216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.825902+0000", 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.72665925300001, 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94761830400000002, 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.628 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223460 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825902+0000", 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825902+0000", 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825900+0000", 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825910+0000", 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4769999999999992e-06 2024-09-14T04:30:52.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825922+0000", 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2763e-05 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.825930+0000", 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2819999999999999e-06 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.826464+0000", 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00053392100000000003 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.826476+0000", 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1878999999999999e-05 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.744094+0000", 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91761810099999996 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.744218+0000", 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012484 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.744347+0000", 2024-09-14T04:30:52.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128768 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761910+0000", 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017563075000000001 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771510+0000", 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095996359999999999 2024-09-14T04:30:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773407+0000", 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018969379999999999 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773454+0000", 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7239e-05 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773520+0000", 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6091000000000001e-05 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223467 13.f 13:f2c42244:::benchmark_data_smithi001_306460_object105223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.831507+0000", 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.721054224, 2024-09-14T04:30:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94215746, 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223467 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831507+0000", 2024-09-14T04:30:52.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831507+0000", 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831506+0000", 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831511+0000", 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4600000000000002e-06 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831526+0000", 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4729000000000001e-05 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831533+0000", 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6699999999999994e-06 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831578+0000", 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4088000000000002e-05 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.831587+0000", 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3219999999999999e-06 2024-09-14T04:30:52.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.747163+0000", 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91557648599999997 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.747300+0000", 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013680300000000001 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.747409+0000", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109009 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761916+0000", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014506633999999999 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.771540+0000", 2024-09-14T04:30:52.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096246960000000003 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773579+0000", 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020389000000000002 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773625+0000", 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5216000000000002e-05 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773664+0000", 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9814999999999997e-05 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223468 13.f 13:fb09172d:::benchmark_data_smithi001_306460_object105224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.832292+0000", 2024-09-14T04:30:52.644 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.720269108, 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94248004100000005, 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223468 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832292+0000", 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832292+0000", 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832291+0000", 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:52.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832296+0000", 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0860000000000001e-06 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832314+0000", 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8070000000000001e-05 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832323+0000", 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1409999999999994e-06 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832598+0000", 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027484100000000001 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.832605+0000", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1019999999999999e-06 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.749959+0000", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91735425699999995 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.750072+0000", 2024-09-14T04:30:52.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112423 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.750181+0000", 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109561 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761921+0000", 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011739233 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.773725+0000", 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011804591 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.774698+0000", 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00097290400000000002 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.774738+0000", 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0077000000000003e-05 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.774772+0000", 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3803999999999997e-05 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223486 13.f 13:fe7aa8b9:::benchmark_data_smithi001_306460_object105242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.853974+0000", 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.69858689700001, 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.933900438, 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223486 2024-09-14T04:30:52.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.853974+0000", 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.853974+0000", 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.853973+0000", 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.853980+0000", 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0230000000000004e-06 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.854011+0000", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0808e-05 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.854020+0000", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.927e-06 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.854186+0000", 2024-09-14T04:30:52.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000165038 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.854192+0000", 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4030000000000004e-06 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.752922+0000", 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.89872985100000002 2024-09-14T04:30:52.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.753041+0000", 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119695 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.753176+0000", 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013489700000000001 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761928+0000", 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087512949999999992 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.774807+0000", 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012879197 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.787781+0000", 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012974121999999999 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.787838+0000", 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6703000000000002e-05 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.787875+0000", 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7020999999999997e-05 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223558 13.f 13:fbd057a6:::benchmark_data_smithi001_306460_object105314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:28.961588+0000", 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.59097293400001, 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.826455991, 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223558 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961588+0000", 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961588+0000", 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961587+0000", 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961597+0000", 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0334e-05 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961611+0000", 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3957e-05 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961618+0000", 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9689999999999997e-06 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961888+0000", 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027050899999999999 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:28.961897+0000", 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.037e-06 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.755847+0000", 2024-09-14T04:30:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.79395008199999995 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.755953+0000", 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105485 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.756055+0000", 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102286 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761932+0000", 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058772039999999996 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.774836+0000", 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012903688999999999 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.787963+0000", 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013127131 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.788009+0000", 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6176e-05 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.788044+0000", 2024-09-14T04:30:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4749000000000002e-05 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223579 13.f 13:fb6652f8:::benchmark_data_smithi001_306460_object105335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:29.000449+0000", 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.552112077, 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.78772918300000005, 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223579 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000449+0000", 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000449+0000", 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000448+0000", 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000454+0000", 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9950000000000002e-06 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000466+0000", 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.218e-05 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000471+0000", 2024-09-14T04:30:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2469999999999999e-06 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000503+0000", 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1365999999999998e-05 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.000508+0000", 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4070000000000001e-06 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.758991+0000", 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.75848302000000001 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.759096+0000", 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105383 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.759210+0000", 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011382099999999999 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.766527+0000", 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073171729999999997 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.676 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.779737+0000", 2024-09-14T04:30:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013209234 2024-09-14T04:30:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.788088+0000", 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083510579999999997 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.788146+0000", 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8712000000000003e-05 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.788178+0000", 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1890000000000001e-05 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:223584 13.f 13:f0af4468:::benchmark_data_smithi001_306460_object105340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2024-09-14T04:30:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:29.016638+0000", 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 143.53592325100001, 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.77287166600000001, 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 223584 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016638+0000", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016638+0000", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016636+0000", 2024-09-14T04:30:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016642+0000", 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.268e-06 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016652+0000", 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0223e-05 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016659+0000", 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9369999999999998e-06 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016698+0000", 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9050000000000001e-05 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.016705+0000", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8990000000000004e-06 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.761957+0000", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.74525154699999996 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.762061+0000", 2024-09-14T04:30:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104546 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.762169+0000", 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010814 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.767025+0000", 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048553880000000004 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.785345+0000", 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018320092999999999 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.789434+0000", 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040889710000000003 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.789476+0000", 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2021000000000003e-05 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:29.789510+0000", 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3757000000000002e-05 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:35186 14.2 14:46474e9a:::benchmark_data_smithi001_337422_object35185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:29:50.481358+0000", 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.071203247, 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.126348766, 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 35186 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481358+0000", 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481358+0000", 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481357+0000", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481410+0000", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2757000000000003e-05 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481425+0000", 2024-09-14T04:30:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5259000000000001e-05 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.481432+0000", 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8410000000000002e-06 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.483177+0000", 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017449799999999999 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.483326+0000", 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014971599999999999 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.483473+0000", 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014705100000000001 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.484712+0000", 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001239001 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.489741+0000", 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050286259999999996 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.607635+0000", 2024-09-14T04:30:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.117894396 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.607679+0000", 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3711000000000002e-05 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.607707+0000", 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7566e-05 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:35204 14.2 14:4738f543:::benchmark_data_smithi001_337422_object35203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:29:50.496002+0000", 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.056559231000001, 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.117145208, 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 35204 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496002+0000", 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496002+0000", 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496000+0000", 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496050+0000", 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9302999999999999e-05 2024-09-14T04:30:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496063+0000", 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2855e-05 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496071+0000", 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9030000000000004e-06 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496118+0000", 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7546999999999997e-05 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496272+0000", 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153852 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:30:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.496424+0000", 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151588 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.497808+0000", 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013846100000000001 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.498081+0000", 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027274200000000003 2024-09-14T04:30:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.613060+0000", 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.114979492 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.613113+0000", 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2259999999999998e-05 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:29:50.613147+0000", 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4490999999999999e-05 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:89928 14.2 14:48fdcb04:::benchmark_data_smithi001_337422_object89927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e517)", 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:47.739097+0000", 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.8134640950000005, 2024-09-14T04:30:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076461319, 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 89928 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739097+0000", 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739097+0000", 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739096+0000", 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:30:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739155+0000", 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8715999999999998e-05 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739165+0000", 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7960000000000007e-06 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.739171+0000", 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6089999999999997e-06 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.753270+0000", 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014098288 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.753431+0000", 2024-09-14T04:30:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016131899999999999 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.753622+0000", 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019130800000000001 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.766710+0000", 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013087942 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.766988+0000", 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000277543 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.811501+0000", 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044513687000000003 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.811608+0000", 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106257 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.815558+0000", 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039506369999999999 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46352.0:89936 14.2 14:4cba26c7:::benchmark_data_smithi001_337422_object89935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e517)", 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:47.744946+0000", 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.807615534, 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073662601999999994, 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46352", 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 89936 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:30:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.744946+0000", 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.744946+0000", 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.744944+0000", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.744994+0000", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0389999999999997e-05 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.745013+0000", 2024-09-14T04:30:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8549000000000001e-05 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.745021+0000", 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4169999999999992e-06 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.757621+0000", 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012600293 2024-09-14T04:30:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.758028+0000", 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040685299999999999 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.758499+0000", 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000470827 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.766900+0000", 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084005360000000001 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.767011+0000", 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111283 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.815663+0000", 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048652497000000003 2024-09-14T04:30:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.815942+0000", 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027814400000000003 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:47.818608+0000", 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002666613 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:30:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:30:52.718 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:30:52.719 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:52.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:52.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:30:52.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:52.864 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:52.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:52.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:52.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:52.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:53.003 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:53.105 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:53.156 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:53.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:53.305 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:53.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:53.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:53.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:53.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:53.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 82 16 94388 94372 71.9208 44.6875 0.0345388 0.0138971 2024-09-14T04:30:53.547 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:53.576 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:53.639 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:53.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:53.713 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:53.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:53.735 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 14 pg_num to 26 2024-09-14T04:30:53.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:53.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:53.873 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:30:53.955 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:53.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/625089883' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[107399]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 50 MiB/s wr, 795 op/s; 27456/275718 objects degraded (9.958%); 4548/275718 objects misplaced (1.650%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/625089883' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:53.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:53 smithi001 ceph-mon[112165]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 50 MiB/s wr, 795 op/s; 27456/275718 objects degraded (9.958%); 4548/275718 objects misplaced (1.650%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:53.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:53.994 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:53.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:54.095 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:53 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/625089883' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T04:30:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:30:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:53 smithi012 ceph-mon[101180]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 50 MiB/s wr, 795 op/s; 27456/275718 objects degraded (9.958%); 4548/275718 objects misplaced (1.650%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:30:54.134 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:54.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:54.276 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:54.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:54.420 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:54.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 83 16 95176 95160 71.6476 49.25 0.032258 0.0139518 2024-09-14T04:30:54.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:54.555 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:54.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:54.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:54.679 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:54.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:54.804 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:54.906 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:54.953 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:54.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:54.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:54 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:30:54.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:54 smithi012 ceph-mon[101180]: osdmap e518: 8 total, 8 up, 6 in 2024-09-14T04:30:54.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27456/275718 objects degraded (9.958%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:55.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:30:55.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[107399]: osdmap e518: 8 total, 8 up, 6 in 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27456/275718 objects degraded (9.958%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[112165]: osdmap e518: 8 total, 8 up, 6 in 2024-09-14T04:30:55.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27456/275718 objects degraded (9.958%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.126 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:55.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:55.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:55.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.314 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:55.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:55.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:55.443 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:55.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 84 16 95704 95688 71.1875 33 0.0142998 0.0140411 2024-09-14T04:30:55.452 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:30:55.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:55.594 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:55.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:55.736 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:55.838 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:55.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:55.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:55.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:55 smithi001 ceph-mon[112165]: pgmap v1639: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/283458 objects degraded (9.686%); 4548/283458 objects misplaced (1.604%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:30:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:55 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:30:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:55 smithi001 ceph-mon[107399]: pgmap v1639: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/283458 objects degraded (9.686%); 4548/283458 objects misplaced (1.604%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:30:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:55 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:30:56.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:56.060 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:56.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:55 smithi012 ceph-mon[101180]: pgmap v1639: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 49 MiB/s wr, 788 op/s; 27456/283458 objects degraded (9.686%); 4548/283458 objects misplaced (1.604%); 3.8 MiB/s, 61 objects/s recovering 2024-09-14T04:30:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:55 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T04:30:56.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:56.184 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:56.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:56.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:56.411 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.433 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:56.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 85 16 95942 95926 70.525 14.875 0.273774 0.0140716 2024-09-14T04:30:56.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:56.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:56.658 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:56.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:56.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:56.816 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:56.918 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:56.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:56.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:56.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:56 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:30:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:56 smithi001 ceph-mon[107399]: osdmap e519: 8 total, 8 up, 6 in 2024-09-14T04:30:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:56 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:30:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:56 smithi001 ceph-mon[112165]: osdmap e519: 8 total, 8 up, 6 in 2024-09-14T04:30:56.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:30:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:30:56] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.51.0" 2024-09-14T04:30:57.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:57.069 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:57.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:56 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T04:30:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:56 smithi012 ceph-mon[101180]: osdmap e519: 8 total, 8 up, 6 in 2024-09-14T04:30:57.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:57.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:57.296 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:57.318 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:57.420 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:57.442 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:57.450 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 86 16 96001 95985 69.7478 3.6875 0.00256999 0.0141881 2024-09-14T04:30:57.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.569 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:57.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:57.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.694 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:57.795 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:57.823 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:57.880 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:30:57.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:57.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:57.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:58.087 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:57 smithi012 ceph-mon[101180]: osdmap e520: 8 total, 8 up, 6 in 2024-09-14T04:30:58.087 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:57 smithi012 ceph-mon[101180]: pgmap v1642: 119 pgs: 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s; 27911/286527 objects degraded (9.741%); 3354/286527 objects misplaced (1.171%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:30:58.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:58.145 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:58.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:57 smithi001 ceph-mon[107399]: osdmap e520: 8 total, 8 up, 6 in 2024-09-14T04:30:58.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:57 smithi001 ceph-mon[107399]: pgmap v1642: 119 pgs: 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s; 27911/286527 objects degraded (9.741%); 3354/286527 objects misplaced (1.171%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:30:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:57 smithi001 ceph-mon[112165]: osdmap e520: 8 total, 8 up, 6 in 2024-09-14T04:30:58.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:57 smithi001 ceph-mon[112165]: pgmap v1642: 119 pgs: 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s; 27911/286527 objects degraded (9.741%); 3354/286527 objects misplaced (1.171%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:30:58.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:58.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:58.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:58.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:58.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 87 16 96473 96457 69.2852 29.5 0.00532489 0.014354 2024-09-14T04:30:58.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:30:58.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:30:58.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:30:58.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:30:58.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:30:58.811 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:30:58.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[107399]: 14.19 scrub starts 2024-09-14T04:30:58.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[107399]: 14.5 scrub starts 2024-09-14T04:30:58.866 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[107399]: osdmap e521: 8 total, 8 up, 6 in 2024-09-14T04:30:58.866 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[112165]: 14.19 scrub starts 2024-09-14T04:30:58.867 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[112165]: 14.5 scrub starts 2024-09-14T04:30:58.867 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:58 smithi001 ceph-mon[112165]: osdmap e521: 8 total, 8 up, 6 in 2024-09-14T04:30:58.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:30:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:58.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:58.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:30:59.061 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:59.091 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:58 smithi012 ceph-mon[101180]: 14.19 scrub starts 2024-09-14T04:30:59.092 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:58 smithi012 ceph-mon[101180]: 14.5 scrub starts 2024-09-14T04:30:59.092 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:58 smithi012 ceph-mon[101180]: osdmap e521: 8 total, 8 up, 6 in 2024-09-14T04:30:59.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:59.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:59.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:59.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:30:59.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:30:59.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 88 16 97145 97129 68.9751 42 0.0315679 0.0144927 2024-09-14T04:30:59.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:30:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.544 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:30:59.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:30:59.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:59.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:59.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:59.779 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:30:59.792 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 7, 6] out_osds: [2, 1] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:30:59.792 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:30:59.792 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-14T04:30:59.792 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:30:59.793 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:30:59.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-14T04:30:59.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:30:59.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:30:59.940 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:30:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:30:59.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:30:59.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[107399]: 14.15 scrub starts 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[107399]: osdmap e522: 8 total, 8 up, 6 in 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 27911/286527 objects degraded (9.741%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[107399]: pgmap v1645: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 18 MiB/s wr, 288 op/s; 27911/286920 objects degraded (9.728%); 3354/286920 objects misplaced (1.169%); 4.1 MiB/s, 64 objects/s recovering 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[112165]: 14.15 scrub starts 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[112165]: osdmap e522: 8 total, 8 up, 6 in 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 27911/286527 objects degraded (9.741%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:30:59.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:30:59 smithi001 ceph-mon[112165]: pgmap v1645: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 18 MiB/s wr, 288 op/s; 27911/286920 objects degraded (9.728%); 3354/286920 objects misplaced (1.169%); 4.1 MiB/s, 64 objects/s recovering 2024-09-14T04:31:00.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:00.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:59 smithi012 ceph-mon[101180]: 14.15 scrub starts 2024-09-14T04:31:00.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:59 smithi012 ceph-mon[101180]: osdmap e522: 8 total, 8 up, 6 in 2024-09-14T04:31:00.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 27911/286527 objects degraded (9.741%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:00.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:30:59 smithi012 ceph-mon[101180]: pgmap v1645: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 9 unknown, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.8 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 18 MiB/s wr, 288 op/s; 27911/286920 objects degraded (9.728%); 3354/286920 objects misplaced (1.169%); 4.1 MiB/s, 64 objects/s recovering 2024-09-14T04:31:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:00.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:00.201 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:00.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:00.336 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:00.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:00.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 89 16 98024 98008 68.8174 54.9375 0.00913664 0.0145252 2024-09-14T04:31:00.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:00.494 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:00.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.626 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:00.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:00.648 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:31:00.648 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPushReply(14.f 522/510 [PushReplyOp(14:f52f43cd:::benchmark_data_smithi001_337422_object5208:head)])", 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.622553+0000", 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.015204530000000001, 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015237530000000001, 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622553+0000", 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622553+0000", 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622553+0000", 2024-09-14T04:31:00.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622557+0000", 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8210000000000003e-06 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622563+0000", 2024-09-14T04:31:00.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4160000000000001e-06 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.622568+0000", 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9899999999999997e-06 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.627558+0000", 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049894370000000002 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.652 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:98170 14.b e522/510 14:dc443633:::benchmark_data_smithi001_337422_object98169:head v 522'6214, mlcod=0'0)", 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.625108+0000", 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012649552999999999, 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012748008, 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625108+0000", 2024-09-14T04:31:00.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625108+0000", 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625107+0000", 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625150+0000", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3136000000000002e-05 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625172+0000", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1696999999999999e-05 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625181+0000", 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5100000000000004e-06 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625283+0000", 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101221 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.625311+0000", 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8245000000000001e-05 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:98171 14.b e522/510 14:d79b57c1:::benchmark_data_smithi001_337422_object98170:head v 522'6215, mlcod=0'0)", 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.628156+0000", 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0096019969999999993, 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0097602959999999999, 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628156+0000", 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628156+0000", 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628155+0000", 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628194+0000", 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8936000000000001e-05 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628211+0000", 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7873999999999999e-05 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628221+0000", 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9860000000000006e-06 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628319+0000", 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7590999999999994e-05 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.628345+0000", 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5885999999999999e-05 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:98175 14.b e522/510 14:db2438bb:::benchmark_data_smithi001_337422_object98174:head v 522'6216, mlcod=0'0)", 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.630698+0000", 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0070598240000000001, 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00727694, 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630698+0000", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630698+0000", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630697+0000", 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:00.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630733+0000", 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6449999999999998e-05 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630753+0000", 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9473000000000001e-05 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.630761+0000", 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7779999999999996e-06 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.631161+0000", 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039934999999999998 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.631187+0000", 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6091000000000001e-05 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.664 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:98178 14.b e522/510)", 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.635157+0000", 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0026012589999999999, 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028960420000000001, 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635157+0000", 2024-09-14T04:31:00.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635157+0000", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635155+0000", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635231+0000", 2024-09-14T04:31:00.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5850000000000001e-05 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635257+0000", 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5397000000000001e-05 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.635266+0000", 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6069999999999997e-06 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:98177 14.4 e522/519 14:2638161f:::benchmark_data_smithi001_337422_object98176:head v 522'5986, mlcod=522'5984)", 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:31:00.636879+0000", 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00087863999999999998, 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001223853, 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636879+0000", 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636879+0000", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636878+0000", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636905+0000", 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6288999999999999e-05 2024-09-14T04:31:00.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636916+0000", 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0998e-05 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636923+0000", 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8950000000000001e-06 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:00.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.636994+0000", 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1484999999999997e-05 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:31:00.637019+0000", 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4824000000000002e-05 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:00.672 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-14T04:31:00.672 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 6 2024-09-14T04:31:00.672 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:31:00.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:00.763 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:00.831 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:31:00.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:00.917 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:31:00.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:00.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:01.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:01.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3232056940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.042 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:00 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3232056940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.043 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:00 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.055 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:01.091 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:31:01.091 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:31:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3232056940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:00 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:01.116 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:31:01.117 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:31:01.145 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:31:01.145 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:31:01.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:01.165 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:31:01.165 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:31:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:01.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:01.198 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:31:01.198 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:31:01.222 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:31:01.222 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:31:01.258 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:31:01.259 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:31:01.289 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:31:01.289 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:31:01.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:01.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.331 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:01.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:01.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:01.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout: 90 3 98943 98940 68.6999 58.25 0.0353908 0.0145504 2024-09-14T04:31:01.459 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:01.561 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:01.584 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:01.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:01.709 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:01.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:01.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:01.939 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:01.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:01.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:02.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:02.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:01 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:01 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:01 smithi012 ceph-mon[101180]: osdmap e523: 8 total, 8 up, 6 in 2024-09-14T04:31:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2169617191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:01 smithi012 ceph-mon[101180]: pgmap v1647: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 1 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 21176/292401 objects degraded (7.242%); 9529/292401 objects misplaced (3.259%); 11 MiB/s, 170 objects/s recovering 2024-09-14T04:31:02.188 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:02.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:02.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:02.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[107399]: osdmap e523: 8 total, 8 up, 6 in 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2169617191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[107399]: pgmap v1647: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 1 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 21176/292401 objects degraded (7.242%); 9529/292401 objects misplaced (3.259%); 11 MiB/s, 170 objects/s recovering 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[112165]: osdmap e523: 8 total, 8 up, 6 in 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2169617191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:02.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:01 smithi001 ceph-mon[112165]: pgmap v1647: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 1 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 21176/292401 objects degraded (7.242%); 9529/292401 objects misplaced (3.259%); 11 MiB/s, 170 objects/s recovering 2024-09-14T04:31:02.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:02.337 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:02.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:02.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total time run: 90.0232 2024-09-14T04:31:02.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Total writes made: 98943 2024-09-14T04:31:02.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Write size: 65536 2024-09-14T04:31:02.451 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Object size: 65536 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Bandwidth (MB/sec): 68.6927 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Bandwidth: 39.6924 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max bandwidth (MB/sec): 216.25 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average IOPS: 1099 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev IOPS: 635.078 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max IOPS: 3460 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min IOPS: 0 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Average Latency(s): 0.014551 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Stddev Latency(s): 0.0480739 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Max latency(s): 3.01332 2024-09-14T04:31:02.452 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Min latency(s): 0.00130141 2024-09-14T04:31:02.456 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:31:02.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:02.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:02.587 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:02.689 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:02.713 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:02.815 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:02.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:02.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:02.998 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:02.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:03.099 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:03.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:03.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:03.273 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:03.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:03.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:03.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:03.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:03.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:03.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:03.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:03.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:03.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:03.880 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:31:03.928 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:03.958 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:03.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:03.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:04.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:04.105 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:04.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:04.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:04.349 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:04.376 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:04.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:04.524 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:04.549 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:04.626 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:04.627 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:31:04.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:04.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:31:04.760 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:04.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:04.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:04.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:04.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:04 smithi001 ceph-mon[107399]: pgmap v1648: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 24 MiB/s wr, 384 op/s; 21655/293451 objects degraded (7.379%); 9529/293451 objects misplaced (3.247%); 5.9 MiB/s, 93 objects/s recovering 2024-09-14T04:31:04.917 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 21655/293451 objects degraded (7.379%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:31:04.917 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:04 smithi001 ceph-mon[112165]: pgmap v1648: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 24 MiB/s wr, 384 op/s; 21655/293451 objects degraded (7.379%); 9529/293451 objects misplaced (3.247%); 5.9 MiB/s, 93 objects/s recovering 2024-09-14T04:31:04.917 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 21655/293451 objects degraded (7.379%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:31:04.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:04.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:05.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:05.056 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:05.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:04 smithi012 ceph-mon[101180]: pgmap v1648: 119 pgs: 11 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 24 MiB/s wr, 384 op/s; 21655/293451 objects degraded (7.379%); 9529/293451 objects misplaced (3.247%); 5.9 MiB/s, 93 objects/s recovering 2024-09-14T04:31:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 21655/293451 objects degraded (7.379%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:31:05.158 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:05.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:05.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:05.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:05.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:05.471 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:05.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:05.617 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:05.719 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:05.754 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:05.856 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:05.884 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:05.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:05.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:05.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:05.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:06.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:06.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:06.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:06.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:06.287 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:06.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:06.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.429 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:06.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:06.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:06.568 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:06.670 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:06.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:06.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:06.812 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:06.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:06.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:06] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[107399]: pgmap v1649: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 26 MiB/s wr, 590 op/s; 20781/292533 objects degraded (7.104%); 9264/292533 objects misplaced (3.167%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[112165]: pgmap v1649: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 26 MiB/s wr, 590 op/s; 20781/292533 objects degraded (7.104%); 9264/292533 objects misplaced (3.167%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:06 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.989 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:06 smithi012 ceph-mon[101180]: pgmap v1649: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 26 MiB/s wr, 590 op/s; 20781/292533 objects degraded (7.104%); 9264/292533 objects misplaced (3.167%); 12 MiB/s, 185 objects/s recovering 2024-09-14T04:31:06.990 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:06 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.990 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:06 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:06.997 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:06.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:07.097 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:07.098 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:07.142 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:07.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:07.274 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:07.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:07.418 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:07.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:07.552 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:07.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:07.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:07.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:07.852 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:07.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:07.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:08.007 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:08.109 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:08.142 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:08.244 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:08.268 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:31:08.276 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:08.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:08.411 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:08.513 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:08.544 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:08.646 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:08.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:08.687 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:31:08.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:08.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:08 smithi012 ceph-mon[101180]: pgmap v1650: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 24 MiB/s wr, 636 op/s; 20781/290082 objects degraded (7.164%); 9012/290082 objects misplaced (3.107%); 12 MiB/s, 192 objects/s recovering 2024-09-14T04:31:08.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:08 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:31:08.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:08.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:08.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:08.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:08.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:08 smithi001 ceph-mon[107399]: pgmap v1650: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 24 MiB/s wr, 636 op/s; 20781/290082 objects degraded (7.164%); 9012/290082 objects misplaced (3.107%); 12 MiB/s, 192 objects/s recovering 2024-09-14T04:31:08.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:08 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:31:08.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:08 smithi001 ceph-mon[112165]: pgmap v1650: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 24 MiB/s wr, 636 op/s; 20781/290082 objects degraded (7.164%); 9012/290082 objects misplaced (3.107%); 12 MiB/s, 192 objects/s recovering 2024-09-14T04:31:08.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:08 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:31:09.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:09.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:09.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:09.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:09.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:09.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:09.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.436 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:09.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:09.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:09.547 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:31:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:31:09.547 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:31:09.564 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:09.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:09.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:09.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:31:09.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[107399]: osdmap e524: 8 total, 8 up, 6 in 2024-09-14T04:31:09.681 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 20781/290082 objects degraded (7.164%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:09.697 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:09.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:09.799 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:09.830 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:09.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:09.960 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:09.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:09.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:09.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:31:09.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[112165]: osdmap e524: 8 total, 8 up, 6 in 2024-09-14T04:31:09.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 20781/290082 objects degraded (7.164%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:09.994 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:09 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:09.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4197687432' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:31:09.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:09 smithi012 ceph-mon[101180]: osdmap e524: 8 total, 8 up, 6 in 2024-09-14T04:31:09.995 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 20781/290082 objects degraded (7.164%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:10.020 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:31:10.020 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:31:10.042 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:31:10.042 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:31:10.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:10.066 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:31:10.067 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:31:10.090 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:31:10.090 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:31:10.118 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:31:10.120 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:31:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.121 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:10.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:10.142 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:31:10.143 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:31:10.170 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:31:10.171 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:31:10.201 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:31:10.201 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:31:10.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:10.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:10.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:10.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:10.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:10.548 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:10.650 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:10.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:10.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:10.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:10.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:10 smithi012 ceph-mon[101180]: pgmap v1652: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 11 MiB/s wr, 460 op/s; 20781/289275 objects degraded (7.184%); 9012/289275 objects misplaced (3.115%); 8.1 MiB/s, 129 objects/s recovering 2024-09-14T04:31:10.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1101079562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:10.920 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:10.947 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:10.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:10.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:10 smithi001 ceph-mon[107399]: pgmap v1652: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 11 MiB/s wr, 460 op/s; 20781/289275 objects degraded (7.184%); 9012/289275 objects misplaced (3.115%); 8.1 MiB/s, 129 objects/s recovering 2024-09-14T04:31:10.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1101079562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:10 smithi001 ceph-mon[112165]: pgmap v1652: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 116 B/s rd, 11 MiB/s wr, 460 op/s; 20781/289275 objects degraded (7.184%); 9012/289275 objects misplaced (3.115%); 8.1 MiB/s, 129 objects/s recovering 2024-09-14T04:31:10.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1101079562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:11.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:11.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:11.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:11.240 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:11.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:11.382 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:11.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:11.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:11.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:11.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:11.648 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:11.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:11.789 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:11.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:11.919 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:11.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:12.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:12.064 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:12.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:12.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.199 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:12.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:12.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:12.344 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:12.446 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:12.476 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:12.577 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:12.609 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:12.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:12.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:12.752 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:12.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:31:12.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:12 smithi012 ceph-mon[101180]: pgmap v1653: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.4 MiB/s wr, 734 op/s; 18983/279372 objects degraded (6.795%); 8673/279372 objects misplaced (3.104%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:12.854 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:12.894 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:12.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:12 smithi001 ceph-mon[107399]: pgmap v1653: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.4 MiB/s wr, 734 op/s; 18983/279372 objects degraded (6.795%); 8673/279372 objects misplaced (3.104%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:12 smithi001 ceph-mon[112165]: pgmap v1653: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.4 MiB/s wr, 734 op/s; 18983/279372 objects degraded (6.795%); 8673/279372 objects misplaced (3.104%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:12.996 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.024 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:13.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:13.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:13.177 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:13.281 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:13.314 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:13.416 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:13.441 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:13.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:13.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:13.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:13.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.726 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:13.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:13.750 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:31:13.751 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:31:13.751 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:31:13.751 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:31:13.751 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:31:13.816 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:13.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:13.867 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:13.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:13.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:13.946 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:13 smithi001 ceph-mon[107399]: pgmap v1654: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 755 op/s; 18983/277704 objects degraded (6.836%); 8673/277704 objects misplaced (3.123%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:13.946 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:13 smithi001 ceph-mon[112165]: pgmap v1654: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 755 op/s; 18983/277704 objects degraded (6.836%); 8673/277704 objects misplaced (3.123%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:13.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:14.011 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:13 smithi012 ceph-mon[101180]: pgmap v1654: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 755 op/s; 18983/277704 objects degraded (6.836%); 8673/277704 objects misplaced (3.123%); 17 MiB/s, 276 objects/s recovering 2024-09-14T04:31:14.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:14.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:14.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:14.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:14.396 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:14.424 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:14.526 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:14.566 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:14.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.710 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:14.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:14.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.854 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:14.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:14.955 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:14.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 18983/277704 objects degraded (6.836%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:14.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 18983/277704 objects degraded (6.836%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:14.982 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:14.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:15.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:15.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 18983/277704 objects degraded (6.836%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:15.107 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:15.209 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.239 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:15.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:15.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:15.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:15.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:15.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:15.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:15.654 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:15.658 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:31:15.757 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:15.793 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:15.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:15 smithi012 ceph-mon[101180]: pgmap v1655: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 979 KiB/s wr, 796 op/s; 17252/269547 objects degraded (6.400%); 8457/269547 objects misplaced (3.137%); 21 MiB/s, 333 objects/s recovering 2024-09-14T04:31:15.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:15.935 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:15.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:15.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:15.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:15 smithi001 ceph-mon[107399]: pgmap v1655: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 979 KiB/s wr, 796 op/s; 17252/269547 objects degraded (6.400%); 8457/269547 objects misplaced (3.137%); 21 MiB/s, 333 objects/s recovering 2024-09-14T04:31:15.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:15 smithi001 ceph-mon[112165]: pgmap v1655: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 979 KiB/s wr, 796 op/s; 17252/269547 objects degraded (6.400%); 8457/269547 objects misplaced (3.137%); 21 MiB/s, 333 objects/s recovering 2024-09-14T04:31:16.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:16.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:31:16.063 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:16.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:16.194 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:16.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:16.360 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:16.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:16.510 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:16.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:16.650 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:31:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.689 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:16.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:16.745 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 7, 6] out_osds: [2, 1] dead_osds: [] live_osds: [1, 3, 4, 6, 7, 0, 2, 5] 2024-09-14T04:31:16.746 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:31:16.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:16.845 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:16.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:16.877 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:16.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:16] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-14T04:31:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:17.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:17.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:17.168 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:17.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:17.270 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:17.301 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:17.404 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:17.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:17.546 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:17.576 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:17.678 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:17.709 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:17.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:17.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:17.847 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:17.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:17.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:17.980 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:17.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:18.040 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:18.041 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:18.041 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:18.041 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:18.041 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:18.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:18.127 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:18.229 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:18.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:18.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:18.398 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:18.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:18.537 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:18.639 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.676 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:18.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:18.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:18 smithi001 ceph-mon[107399]: pgmap v1656: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 17252/265719 objects degraded (6.493%); 8300/265719 objects misplaced (3.124%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:18.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:18 smithi001 ceph-mon[112165]: pgmap v1656: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 17252/265719 objects degraded (6.493%); 8300/265719 objects misplaced (3.124%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:18.748 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:31:18.748 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:31:18.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:31:18.748 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45647.0:243548 12.7 12:ed83732d:::benchmark_data_smithi001_255951_object97888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 548.34270463300004, 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96499681999999998, 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45647", 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 243548 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.749 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401734+0000", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401731+0000", 2024-09-14T04:31:18.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401744+0000", 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2456999999999999e-05 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401774+0000", 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0086e-05 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401787+0000", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3644e-05 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401910+0000", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012274000000000001 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:10.401918+0000", 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2024-09-14T04:31:18.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.361975+0000", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96005681700000001 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362097+0000", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012235999999999999 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.362195+0000", 2024-09-14T04:31:18.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8171000000000006e-05 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.365736+0000", 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035408940000000002 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366255+0000", 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051849299999999997 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366541+0000", 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028677599999999998 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366603+0000", 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2151999999999997e-05 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:22:11.366730+0000", 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012696499999999999 2024-09-14T04:31:18.755 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:209765 13.7 13:e1e52107:::benchmark_data_smithi001_306460_object91521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:12.732086+0000", 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 186.012352056, 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93554505799999998, 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.756 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209765 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732086+0000", 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732086+0000", 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732085+0000", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732093+0000", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4100000000000008e-06 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732112+0000", 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9293999999999999e-05 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732121+0000", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4980000000000006e-06 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732177+0000", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6206999999999999e-05 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:12.732186+0000", 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0559999999999994e-06 2024-09-14T04:31:18.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.662302+0000", 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93011561499999995 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.662461+0000", 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015971400000000001 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:31:18.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.662915+0000", 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045348699999999998 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.667527+0000", 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004612161 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.667563+0000", 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6013000000000002e-05 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.667571+0000", 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5970000000000003e-06 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.667600+0000", 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9720999999999999e-05 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:13.667631+0000", 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0919e-05 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216014 13.7 13:e2b4ceb2:::benchmark_data_smithi001_306460_object97770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2024-09-14T04:31:18.763 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:18.684903+0000", 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 180.05953539999999, 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1584663279999998, 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216014 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684903+0000", 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684903+0000", 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684901+0000", 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684908+0000", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2069999999999998e-06 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684921+0000", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2876e-05 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.684928+0000", 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2024-09-14T04:31:18.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.691705+0000", 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067763099999999998 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.691718+0000", 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3648e-05 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248762+0000", 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55704374199999995 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248773+0000", 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1153e-05 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.835182+0000", 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.586408789 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.835287+0000", 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010519899999999999 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.835664+0000", 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037671500000000001 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.838218+0000", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025542809999999998 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.838229+0000", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.134e-05 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843136+0000", 2024-09-14T04:31:18.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049062539999999997 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843184+0000", 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8133999999999997e-05 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843237+0000", 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3136000000000001e-05 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843369+0000", 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132158 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216025 13.7 13:e6fe2036:::benchmark_data_smithi001_306460_object97781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:18.697398+0000", 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 180.04704013899999, 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1464686830000002, 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216025 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697398+0000", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697398+0000", 2024-09-14T04:31:18.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697396+0000", 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697406+0000", 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0428000000000001e-05 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697428+0000", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2314999999999999e-05 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697438+0000", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0081999999999999e-05 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697619+0000", 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018099500000000001 2024-09-14T04:31:18.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.697630+0000", 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.048e-05 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248801+0000", 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55117108400000003 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248809+0000", 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9659999999999996e-06 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.838256+0000", 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5894466700000001 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.838341+0000", 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4826000000000004e-05 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.838702+0000", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036171 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843384+0000", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046813109999999996 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843587+0000", 2024-09-14T04:31:18.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000203434 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843648+0000", 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1452999999999996e-05 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843768+0000", 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119706 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843799+0000", 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0843000000000003e-05 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843867+0000", 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7805000000000006e-05 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216068 13.7 13:e18a4e96:::benchmark_data_smithi001_306460_object97824:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:18.754116+0000", 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.990322553, 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0898990739999999, 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216068 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754116+0000", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754116+0000", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754109+0000", 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999933 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754121+0000", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1994000000000001e-05 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754132+0000", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1105e-05 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754137+0000", 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9779999999999996e-06 2024-09-14T04:31:18.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754179+0000", 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1857000000000003e-05 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.754187+0000", 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.712e-06 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248834+0000", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.49464739699999999 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248842+0000", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1850000000000004e-06 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.840566+0000", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.591724063 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.840700+0000", 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013363399999999999 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.841030+0000", 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032965300000000001 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843596+0000", 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025668449999999999 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843704+0000", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107122 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843734+0000", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0460999999999999e-05 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843904+0000", 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017007200000000001 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.843936+0000", 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1525000000000001e-05 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.844015+0000", 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9158000000000003e-05 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216073 13.7 13:e6a908a2:::benchmark_data_smithi001_306460_object97829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:18.767539+0000", 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.976898949, 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0901759019999999, 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216073 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767539+0000", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767539+0000", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767538+0000", 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:31:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767545+0000", 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3849999999999999e-06 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767556+0000", 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1610000000000001e-05 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767561+0000", 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2399999999999998e-06 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767602+0000", 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0697e-05 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:18.767610+0000", 2024-09-14T04:31:18.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6000000000000001e-06 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248904+0000", 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.48129462699999997 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.248912+0000", 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1550000000000007e-06 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.844042+0000", 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.595129982 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.844120+0000", 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7787000000000006e-05 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.844349+0000", 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022921899999999999 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.851923+0000", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075732789999999996 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.851968+0000", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5148000000000002e-05 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.852046+0000", 2024-09-14T04:31:18.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7871e-05 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857551+0000", 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005505027 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857604+0000", 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3607999999999999e-05 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857715+0000", 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110821 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216101 13.7 13:e4ee7e2a:::benchmark_data_smithi001_306460_object97857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:19.446916+0000", 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.29752195399999, 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4110684660000001, 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.798 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216101 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446916+0000", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446916+0000", 2024-09-14T04:31:18.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446915+0000", 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446924+0000", 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5009999999999993e-06 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446936+0000", 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1708999999999999e-05 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.446943+0000", 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6220000000000003e-06 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.447001+0000", 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8488e-05 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.447012+0000", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0550999999999999e-05 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.846427+0000", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.399414903 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.846544+0000", 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011759799999999999 2024-09-14T04:31:18.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.846908+0000", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036373599999999997 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.852111+0000", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052028669999999999 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.854896+0000", 2024-09-14T04:31:18.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002785647 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857726+0000", 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002829729 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857820+0000", 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4061000000000006e-05 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857860+0000", 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0110000000000001e-05 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857985+0000", 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124399 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216117 13.7 13:e700d45f:::benchmark_data_smithi001_306460_object97873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-14T04:31:18.805 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:19.465429+0000", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.27900880000001, 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3928366640000001, 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216117 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465429+0000", 2024-09-14T04:31:18.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465429+0000", 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465429+0000", 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465435+0000", 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2759999999999997e-06 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465446+0000", 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0555e-05 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465452+0000", 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1650000000000003e-06 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465570+0000", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011865400000000001 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.465582+0000", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1216999999999999e-05 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.849232+0000", 2024-09-14T04:31:18.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3836500940000001 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.849341+0000", 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010932799999999999 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.849743+0000", 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040155200000000001 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.854868+0000", 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051252229999999999 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857732+0000", 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028642099999999998 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857763+0000", 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0871999999999999e-05 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.858046+0000", 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028333900000000002 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.858131+0000", 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5198999999999995e-05 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.858266+0000", 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013465499999999999 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:216127 13.7 13:e2a36855:::benchmark_data_smithi001_306460_object97883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:19.479804+0000", 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 179.26463446299999, 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3812944140000001, 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216127 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479804+0000", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479804+0000", 2024-09-14T04:31:18.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479802+0000", 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479809+0000", 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3149999999999997e-06 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479820+0000", 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.132e-05 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479828+0000", 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5560000000000008e-06 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479885+0000", 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7033000000000001e-05 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:19.479899+0000", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3552e-05 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.852134+0000", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.372235155 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.852251+0000", 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116469 2024-09-14T04:31:18.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.852640+0000", 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038964099999999998 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857769+0000", 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051286079999999998 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.857795+0000", 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.616e-05 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.858317+0000", 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00052148500000000003 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.860893+0000", 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025764009999999999 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.860952+0000", 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8831e-05 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:20.861098+0000", 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014637999999999999 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.820 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:217045 13.e 13:72e021e6:::benchmark_data_smithi001_306460_object98801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:21.744467+0000", 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 176.99997125300001, 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0162150489999999, 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 217045 2024-09-14T04:31:18.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744467+0000", 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744467+0000", 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744466+0000", 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744471+0000", 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3830000000000002e-06 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744482+0000", 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744489+0000", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.503e-06 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744576+0000", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7410000000000005e-05 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.744585+0000", 2024-09-14T04:31:18.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0250000000000008e-06 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.752277+0000", 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.007691012 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.752362+0000", 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5073000000000004e-05 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.752446+0000", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4726000000000002e-05 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760443+0000", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079964370000000003 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760479+0000", 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6304999999999999e-05 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760597+0000", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011837600000000001 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760642+0000", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4379000000000003e-05 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760682+0000", 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0218e-05 2024-09-14T04:31:18.827 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:217062 13.e 13:7b96eb1c:::benchmark_data_smithi001_306460_object98818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:21.764480+0000", 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 176.97995859900001, 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99635458399999999, 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 217062 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764480+0000", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764480+0000", 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764478+0000", 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764483+0000", 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2340000000000002e-06 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764496+0000", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543e-05 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764503+0000", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.092e-06 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764553+0000", 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0095000000000001e-05 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.764561+0000", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.755133+0000", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99057214199999999 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.755261+0000", 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127603 2024-09-14T04:31:18.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.755382+0000", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121001 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760566+0000", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051839659999999999 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760727+0000", 2024-09-14T04:31:18.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000161335 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760768+0000", 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1452000000000002e-05 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760798+0000", 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9757999999999999e-05 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760834+0000", 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6087999999999997e-05 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.835 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:217106 13.e 13:70a36ffb:::benchmark_data_smithi001_306460_object98862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:21.805421+0000", 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 176.93901730799999, 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96397109000000003, 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.836 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 217106 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805421+0000", 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805421+0000", 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805420+0000", 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805425+0000", 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8540000000000002e-06 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805438+0000", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2417e-05 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805443+0000", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3170000000000001e-06 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805491+0000", 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7465e-05 2024-09-14T04:31:18.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.805502+0000", 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1137e-05 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.757767+0000", 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95226563099999995 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.757930+0000", 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000162676 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.758045+0000", 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114453 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760693+0000", 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002648423 2024-09-14T04:31:18.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760882+0000", 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018952699999999999 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769308+0000", 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084251980000000001 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769353+0000", 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4885000000000001e-05 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769392+0000", 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9440999999999997e-05 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.843 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:217119 13.e 13:7752db76:::benchmark_data_smithi001_306460_object98875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:21.817699+0000", 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 176.92673945799999, 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95194425599999999, 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 217119 2024-09-14T04:31:18.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817699+0000", 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817699+0000", 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817697+0000", 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817705+0000", 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8990000000000001e-06 2024-09-14T04:31:18.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817722+0000", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7430000000000001e-05 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817731+0000", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4470000000000006e-06 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817785+0000", 2024-09-14T04:31:18.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4622e-05 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:21.817795+0000", 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6809999999999993e-06 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.760918+0000", 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94312321600000004 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.761021+0000", 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102818 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.761126+0000", 2024-09-14T04:31:18.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105039 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769449+0000", 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083225899999999995 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769506+0000", 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7348e-05 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769543+0000", 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7100000000000001e-05 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769609+0000", 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5951999999999995e-05 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:22.769643+0000", 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4004000000000002e-05 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.851 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229592 13.18 13:1ab7cfb4:::benchmark_data_smithi001_306460_object111348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e482)", 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:45.065287+0000", 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 153.67915096600001, 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0046526849999999, 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229592 2024-09-14T04:31:18.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065287+0000", 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065287+0000", 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065286+0000", 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065291+0000", 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5559999999999998e-06 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065305+0000", 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3575e-05 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065312+0000", 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7839999999999998e-06 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065394+0000", 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2206999999999994e-05 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065402+0000", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0739999999999993e-06 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.065453+0000", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0000514760000001 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.065543+0000", 2024-09-14T04:31:18.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9369000000000004e-05 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.065821+0000", 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027811900000000001 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.069614+0000", 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037930419999999999 2024-09-14T04:31:18.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.069653+0000", 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9530999999999998e-05 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.069844+0000", 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019028900000000001 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.069902+0000", 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8335999999999998e-05 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.069940+0000", 2024-09-14T04:31:18.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7870999999999997e-05 2024-09-14T04:31:18.860 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.860 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.860 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.862 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229593 13.18 13:1dee50a5:::benchmark_data_smithi001_306460_object111349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e482)", 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:45.065328+0000", 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 153.67911064899999, 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0126250379999999, 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.863 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229593 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065328+0000", 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065328+0000", 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065327+0000", 2024-09-14T04:31:18.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065331+0000", 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4489999999999999e-06 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065338+0000", 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065343+0000", 2024-09-14T04:31:18.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0950000000000003e-06 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065432+0000", 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9007999999999999e-05 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.065441+0000", 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6100000000000006e-06 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.067981+0000", 2024-09-14T04:31:18.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.002540078 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.068078+0000", 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7057000000000003e-05 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.068341+0000", 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026269100000000001 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.070001+0000", 2024-09-14T04:31:18.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001660627 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.070007+0000", 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9410000000000004e-06 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.077829+0000", 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0078217770000000002 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.077910+0000", 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0805999999999998e-05 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.077953+0000", 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2718e-05 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.875 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:18 smithi012 ceph-mon[101180]: pgmap v1656: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 17252/265719 objects degraded (6.493%); 8300/265719 objects misplaced (3.124%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:18.875 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229613 13.18 13:1ef28023:::benchmark_data_smithi001_306460_object111369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e482)", 2024-09-14T04:31:18.875 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:45.093887+0000", 2024-09-14T04:31:18.875 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 153.65055127100001, 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98419139600000005, 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229613 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.876 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093887+0000", 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093887+0000", 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093886+0000", 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093892+0000", 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9340000000000003e-06 2024-09-14T04:31:18.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093905+0000", 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3006999999999999e-05 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093913+0000", 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1890000000000007e-06 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093962+0000", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8943000000000002e-05 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.093973+0000", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1219e-05 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.070021+0000", 2024-09-14T04:31:18.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97604827299999997 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.070083+0000", 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1690000000000001e-05 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.070285+0000", 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020201600000000001 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.074969+0000", 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046839639999999997 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.074976+0000", 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9970000000000001e-06 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.077999+0000", 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030225819999999998 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.078040+0000", 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1016999999999997e-05 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.078078+0000", 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8770999999999999e-05 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.883 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229654 13.18 13:1f3fb3c6:::benchmark_data_smithi001_306460_object111410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e482)", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:45.140702+0000", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 153.603735841, 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93752059399999998, 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229654 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140702+0000", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140702+0000", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140701+0000", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140709+0000", 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6590000000000007e-06 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140731+0000", 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2123000000000001e-05 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140739+0000", 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8000000000000004e-06 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140841+0000", 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010182 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.140850+0000", 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4169999999999992e-06 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.072094+0000", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93124417699999995 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.072201+0000", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010733 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.072575+0000", 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037418299999999999 2024-09-14T04:31:18.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.075041+0000", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024659370000000001 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.077968+0000", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029263000000000002 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.078136+0000", 2024-09-14T04:31:18.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016816800000000001 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.078178+0000", 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2548e-05 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:46.078223+0000", 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4651000000000003e-05 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.890 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:95927 14.5 e518/494 14:a02d29f0:::benchmark_data_smithi001_337422_object95926:head v 518'6105, mlcod=518'6102)", 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:55.779835+0000", 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 22.964603574000002, 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.242536789, 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779835+0000", 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779835+0000", 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779834+0000", 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779897+0000", 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3794000000000006e-05 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779911+0000", 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3286e-05 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.779917+0000", 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.494e-06 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.780195+0000", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000277605 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.780238+0000", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2960000000000002e-05 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.022371+0000", 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.242133611 2024-09-14T04:31:18.894 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:95930 14.4 e518/494)", 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:55.797395+0000", 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 22.947043624999999, 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.25483438200000003, 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797395+0000", 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797395+0000", 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797393+0000", 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797425+0000", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1547999999999997e-05 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797447+0000", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2619000000000001e-05 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.797458+0000", 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0757000000000001e-05 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.052203+0000", 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.25474450500000001 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.052229+0000", 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6393e-05 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.898 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.46352.0:95931 14.d e518/514 14:b1604bb5:::benchmark_data_smithi001_337422_object95930:head v 518'6029, mlcod=518'6027)", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:30:55.804303+0000", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 22.940134983, 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.26551101999999999, 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804303+0000", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:31:18.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804303+0000", 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804303+0000", 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804336+0000", 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3710000000000001e-05 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804359+0000", 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2733000000000001e-05 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:55.804370+0000", 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1017e-05 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.022829+0000", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.21845933000000001 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.022852+0000", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2580999999999999e-05 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.069769+0000", 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.046917234000000002 2024-09-14T04:31:18.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:30:56.069814+0000", 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5015999999999997e-05 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:31:18.903 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:31:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:18.905 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:18.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:19.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:19.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:19.178 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:19.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:19.324 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:19.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:19.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:19.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:19.569 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:19.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:19.684 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:31:19.707 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:19.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:19.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:19.882 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:19.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:19.924 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:31:19.955 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:19.955 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 17252/265719 objects degraded (6.493%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:19.955 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4029206781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:19.956 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:19.956 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 17252/265719 objects degraded (6.493%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:19.956 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4029206781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:19.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:20.023 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:20.081 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:20.081 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 17252/265719 objects degraded (6.493%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:20.081 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4029206781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:20.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.162 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:20.263 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:20.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:20.388 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:31:20.388 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:31:20.389 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:31:20.389 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:31:20.389 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:31:20.389 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:31:20.389 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:31:20.390 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:31:20.391 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:31:20.391 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:31:20.391 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:31:20.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:20.429 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:20.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.556 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:20.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:20.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.824 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:20.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:20.969 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:20.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: pgmap v1657: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 17252/264942 objects degraded (6.512%); 8300/264942 objects misplaced (3.133%); 19 MiB/s, 296 objects/s recovering 2024-09-14T04:31:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:31:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: osdmap e525: 8 total, 8 up, 6 in 2024-09-14T04:31:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:20.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:20.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: pgmap v1657: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 17252/264942 objects degraded (6.512%); 8300/264942 objects misplaced (3.133%); 19 MiB/s, 296 objects/s recovering 2024-09-14T04:31:20.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:31:20.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:20.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: osdmap e525: 8 total, 8 up, 6 in 2024-09-14T04:31:20.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:20.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:21.069 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: pgmap v1657: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 17252/264942 objects degraded (6.512%); 8300/264942 objects misplaced (3.133%); 19 MiB/s, 296 objects/s recovering 2024-09-14T04:31:21.069 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:31:21.069 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/928346106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:21.069 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: osdmap e525: 8 total, 8 up, 6 in 2024-09-14T04:31:21.070 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:21.070 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:21.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.110 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:21.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:21.211 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:21.241 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:21.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:21.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:21.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:21.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:21.528 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:21.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:21.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:21.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.807 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:21.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:21.908 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:21.942 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:21.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:22.044 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:22.068 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:22.169 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.204 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:22.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:22.289 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:22.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.348 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:22.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:22.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:22.479 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:22.584 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:22.613 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:22.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:22.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:22.759 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:22.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:22.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:22.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:22.951 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:22 smithi001 ceph-mon[107399]: pgmap v1659: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 15424/255000 objects degraded (6.049%); 7930/255000 objects misplaced (3.110%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:22.951 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:22 smithi001 ceph-mon[112165]: pgmap v1659: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 15424/255000 objects degraded (6.049%); 7930/255000 objects misplaced (3.110%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:23.004 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:23.026 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:22 smithi012 ceph-mon[101180]: pgmap v1659: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 15424/255000 objects degraded (6.049%); 7930/255000 objects misplaced (3.110%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:23.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:23.167 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:23.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:23.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:23.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:23.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:23.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:23.441 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:23.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:23.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.589 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:23.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:23.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:23.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.721 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:23.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:23.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:23.855 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:23.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:23.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:23.999 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:23.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:24.100 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.131 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:24.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:24.210 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:24.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:24.278 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:24.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.424 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:24.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:24.525 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:24.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:24.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.701 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:24.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:24.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.842 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:24.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:24.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:31:24.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:24 smithi001 ceph-mon[107399]: pgmap v1660: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15424/253470 objects degraded (6.085%); 7930/253470 objects misplaced (3.129%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:24.912 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 15424/253470 objects degraded (6.085%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:24.912 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:24 smithi001 ceph-mon[112165]: pgmap v1660: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15424/253470 objects degraded (6.085%); 7930/253470 objects misplaced (3.129%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:24.912 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 15424/253470 objects degraded (6.085%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:24.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:24.985 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:24.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:25.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:24 smithi012 ceph-mon[101180]: pgmap v1660: 119 pgs: 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15424/253470 objects degraded (6.085%); 7930/253470 objects misplaced (3.129%); 20 MiB/s, 327 objects/s recovering 2024-09-14T04:31:25.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 15424/253470 objects degraded (6.085%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:25.124 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:25.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:25.263 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:25.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:25.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:25.405 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:25.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:25.542 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:25.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:25.683 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:25.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:31:25.784 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:25.823 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:25.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:25.925 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:25.955 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:25.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:25.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:26.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:26.095 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:26.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:26.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:26.243 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:26.345 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:26.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:26.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:26.476 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:26.508 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:26.609 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:26.639 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:26.740 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:26.773 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:26.875 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:26.908 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:26.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:26 smithi001 ceph-mon[107399]: pgmap v1661: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 14063/245295 objects degraded (5.733%); 7688/245295 objects misplaced (3.134%); 20 MiB/s, 317 objects/s recovering 2024-09-14T04:31:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:26] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-09-14T04:31:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:26 smithi001 ceph-mon[112165]: pgmap v1661: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 14063/245295 objects degraded (5.733%); 7688/245295 objects misplaced (3.134%); 20 MiB/s, 317 objects/s recovering 2024-09-14T04:31:27.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.043 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:27.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:26 smithi012 ceph-mon[101180]: pgmap v1661: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 14063/245295 objects degraded (5.733%); 7688/245295 objects misplaced (3.134%); 20 MiB/s, 317 objects/s recovering 2024-09-14T04:31:27.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:27.187 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:27.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:27.329 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:27.431 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:27.446 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:27.487 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:27.511 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:31:27.511 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:31:27.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:27.638 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:27.702 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:31:27.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.778 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:27.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:27.879 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:27.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:27.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:27.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:31:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:31:28.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.051 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:28.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:31:28.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:28.182 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:28.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.325 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:28.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:28.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.473 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:28.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:28.574 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:28.601 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:28.665 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:28.703 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:28.713 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[107399]: 14.19 scrub starts 2024-09-14T04:31:28.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[107399]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14063/241266 objects degraded (5.829%); 7491/241266 objects misplaced (3.105%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:28.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:31:28.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[107399]: osdmap e526: 8 total, 8 up, 6 in 2024-09-14T04:31:28.714 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[107399]: 14.19 scrub ok 2024-09-14T04:31:28.742 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:28.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:28.876 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:28.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:28.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[112165]: 14.19 scrub starts 2024-09-14T04:31:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[112165]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14063/241266 objects degraded (5.829%); 7491/241266 objects misplaced (3.105%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:31:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[112165]: osdmap e526: 8 total, 8 up, 6 in 2024-09-14T04:31:28.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:28 smithi001 ceph-mon[112165]: 14.19 scrub ok 2024-09-14T04:31:28.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:29.018 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:29.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:28 smithi012 ceph-mon[101180]: 14.19 scrub starts 2024-09-14T04:31:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:28 smithi012 ceph-mon[101180]: pgmap v1662: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14063/241266 objects degraded (5.829%); 7491/241266 objects misplaced (3.105%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4253170358' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:31:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:28 smithi012 ceph-mon[101180]: osdmap e526: 8 total, 8 up, 6 in 2024-09-14T04:31:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:28 smithi012 ceph-mon[101180]: 14.19 scrub ok 2024-09-14T04:31:29.120 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:29.149 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:29.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:29.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:31:29.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:29.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:29.435 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:29.537 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:29.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.579 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:29.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:29.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:29.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:29.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:29.843 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:29.945 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[112165]: 14.5 scrub starts 2024-09-14T04:31:29.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[112165]: 14.5 scrub ok 2024-09-14T04:31:29.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 14063/241266 objects degraded (5.829%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[107399]: 14.5 scrub starts 2024-09-14T04:31:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[107399]: 14.5 scrub ok 2024-09-14T04:31:29.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 14063/241266 objects degraded (5.829%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:29.975 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:29.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:30.077 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:29 smithi012 ceph-mon[101180]: 14.5 scrub starts 2024-09-14T04:31:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:29 smithi012 ceph-mon[101180]: 14.5 scrub ok 2024-09-14T04:31:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 14063/241266 objects degraded (5.829%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:30.106 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:30.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:30.251 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:30.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:30.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.391 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:30.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:30.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:30.532 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:30.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:30.673 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:30.775 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:30.808 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:30.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:30.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:30.952 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:30.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:30.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[112165]: 14.15 scrub starts 2024-09-14T04:31:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[112165]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 14063/240408 objects degraded (5.850%); 7491/240408 objects misplaced (3.116%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:30.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[112165]: 14.15 scrub ok 2024-09-14T04:31:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[107399]: 14.15 scrub starts 2024-09-14T04:31:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[107399]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 14063/240408 objects degraded (5.850%); 7491/240408 objects misplaced (3.116%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:30.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:30 smithi001 ceph-mon[107399]: 14.15 scrub ok 2024-09-14T04:31:31.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:31.091 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:30 smithi012 ceph-mon[101180]: 14.15 scrub starts 2024-09-14T04:31:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:30 smithi012 ceph-mon[101180]: pgmap v1664: 119 pgs: 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 14063/240408 objects degraded (5.850%); 7491/240408 objects misplaced (3.116%); 20 MiB/s, 323 objects/s recovering 2024-09-14T04:31:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:30 smithi012 ceph-mon[101180]: 14.15 scrub ok 2024-09-14T04:31:31.193 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:31.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:31.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:31.355 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:31.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:31.498 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:31.599 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:31.631 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:31.732 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:31.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:31:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.776 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:31.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:31.877 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:31.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:31.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:31.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:31 smithi001 ceph-mon[112165]: pgmap v1665: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 12982/229353 objects degraded (5.660%); 6994/229353 objects misplaced (3.049%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:31.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:31 smithi001 ceph-mon[107399]: pgmap v1665: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 12982/229353 objects degraded (5.660%); 6994/229353 objects misplaced (3.049%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:32.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:32.045 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:32.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:31 smithi012 ceph-mon[101180]: pgmap v1665: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 12982/229353 objects degraded (5.660%); 6994/229353 objects misplaced (3.049%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:32.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:32.190 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:32.231 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:32.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:32.330 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:32.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:32.432 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:32.465 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:32.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:32.606 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:32.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:32.708 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:32.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:32.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:32.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:32.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:32.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:32.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:33.020 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:33.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:33.150 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:33.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:33.296 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:33.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:33.439 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:33.540 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:33.583 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:33.685 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:33.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:33.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:33.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:33.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:33.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:33.993 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:33.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:33.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:34.094 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:34.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:34.120 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:31:34.121 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:31:34.137 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:34.146 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:31:34.147 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:31:34.171 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:31:34.172 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:31:34.195 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:31:34.196 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:31:34.233 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:31:34.234 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:31:34.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:34.262 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:31:34.262 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:31:34.280 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:34.296 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:31:34.296 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:31:34.324 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:31:34.324 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:31:34.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:34.425 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:34.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:34.527 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:34.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:34.659 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:34.674 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:31:34.692 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:34.794 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:34.826 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:34.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:34.964 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:34.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:34.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[112165]: pgmap v1666: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 12982/227685 objects degraded (5.702%); 6994/227685 objects misplaced (3.072%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:31:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2677812247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:34.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 12982/227685 objects degraded (5.702%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:34.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[107399]: pgmap v1666: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 12982/227685 objects degraded (5.702%); 6994/227685 objects misplaced (3.072%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:31:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2677812247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:34.973 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 12982/227685 objects degraded (5.702%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:35.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:34 smithi012 ceph-mon[101180]: pgmap v1666: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 12982/227685 objects degraded (5.702%); 6994/227685 objects misplaced (3.072%); 20 MiB/s, 312 objects/s recovering 2024-09-14T04:31:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:34 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:31:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:34 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2677812247' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 12982/227685 objects degraded (5.702%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:35.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:35.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:35.250 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:35.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:35.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:35.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:35.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:35.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:35.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:35.622 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:35.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:35.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:35.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:35.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:35.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:35.929 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:35.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:35.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[112165]: osdmap e527: 8 total, 8 up, 6 in 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[107399]: osdmap e527: 8 total, 8 up, 6 in 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:36.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:36.072 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:36.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:35 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:35 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/271308181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:31:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:35 smithi012 ceph-mon[101180]: osdmap e527: 8 total, 8 up, 6 in 2024-09-14T04:31:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:31:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:36.174 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:36.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:36.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.337 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:36.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:36.438 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:36.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.469 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:36.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:36.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:36.593 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:36.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:36.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:36.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:36.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:36.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:36.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:36 smithi001 ceph-mon[112165]: pgmap v1668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 11830/218340 objects degraded (5.418%); 6753/218340 objects misplaced (3.093%); 23 MiB/s, 371 objects/s recovering 2024-09-14T04:31:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:36 smithi001 ceph-mon[107399]: pgmap v1668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 11830/218340 objects degraded (5.418%); 6753/218340 objects misplaced (3.093%); 23 MiB/s, 371 objects/s recovering 2024-09-14T04:31:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:36] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.51.0" 2024-09-14T04:31:36.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:36.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:36 smithi012 ceph-mon[101180]: pgmap v1668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 11830/218340 objects degraded (5.418%); 6753/218340 objects misplaced (3.093%); 23 MiB/s, 371 objects/s recovering 2024-09-14T04:31:37.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:37.141 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:37.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:37.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:37.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:37.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:37.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:37.560 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:37.666 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.698 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:37.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:37.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:37.837 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:37.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:37.979 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:37.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:37.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:38.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:38.111 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:38.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:38.253 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:38.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:38.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:38.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:38.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:38.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:38.687 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:38.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:38.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:38.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:38.950 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:38 smithi012 ceph-mon[101180]: pgmap v1669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213969 objects degraded (5.567%); 6451/213969 objects misplaced (3.015%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:38.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:38.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:38.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:38 smithi001 ceph-mon[107399]: pgmap v1669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213969 objects degraded (5.567%); 6451/213969 objects misplaced (3.015%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:38.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:38 smithi001 ceph-mon[112165]: pgmap v1669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213969 objects degraded (5.567%); 6451/213969 objects misplaced (3.015%); 20 MiB/s, 321 objects/s recovering 2024-09-14T04:31:39.070 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:39.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:39.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:39.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:39.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:39.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:39.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:39.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:39.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:39.506 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:39.608 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:39.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:39.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:39.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:39.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:39.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:39.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:39.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:39.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 11912/213969 objects degraded (5.567%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 11912/213969 objects degraded (5.567%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:40.032 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:40.061 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:40.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 11912/213969 objects degraded (5.567%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:40.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:40.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.204 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:40.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:40.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:40.333 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:40.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:40.463 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:40.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:40.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:40.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:40.723 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:40.825 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:40.851 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:40.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:40.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:40.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:40 smithi001 ceph-mon[107399]: pgmap v1670: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213171 objects degraded (5.588%); 6451/213171 objects misplaced (3.026%); 20 MiB/s, 318 objects/s recovering 2024-09-14T04:31:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:40 smithi001 ceph-mon[112165]: pgmap v1670: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213171 objects degraded (5.588%); 6451/213171 objects misplaced (3.026%); 20 MiB/s, 318 objects/s recovering 2024-09-14T04:31:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:40.992 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:40.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:41.055 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:40 smithi012 ceph-mon[101180]: pgmap v1670: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 11912/213171 objects degraded (5.588%); 6451/213171 objects misplaced (3.026%); 20 MiB/s, 318 objects/s recovering 2024-09-14T04:31:41.093 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:41.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:41.227 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:41.264 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:41.366 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:41.396 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:41.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:41.539 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:41.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:41.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.685 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:41.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:41.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:41.822 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:41.918 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:31:41.918 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:31:41.918 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:31:41.918 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:31:41.918 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:31:41.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:41.965 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:41.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:42.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:42.103 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:42.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:42.244 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:42.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:42.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:42.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:42.520 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:42.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:42.660 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:42.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:42.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:42.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:42.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:42.936 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:42.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:42.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:42 smithi001 ceph-mon[107399]: pgmap v1671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 10850/202824 objects degraded (5.349%); 5916/202824 objects misplaced (2.917%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:42.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:42 smithi001 ceph-mon[112165]: pgmap v1671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 10850/202824 objects degraded (5.349%); 5916/202824 objects misplaced (2.917%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:43.037 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:43.066 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:42 smithi012 ceph-mon[101180]: pgmap v1671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 10850/202824 objects degraded (5.349%); 5916/202824 objects misplaced (2.917%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:43.111 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:31:43.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:43.202 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:43.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:31:43.304 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:43.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:43.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:43.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:43.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:31:43.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:43.641 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:43.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:43.786 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:43.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:43.924 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:43.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:43.948 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:43 smithi001 ceph-mon[112165]: pgmap v1672: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 10850/201192 objects degraded (5.393%); 5916/201192 objects misplaced (2.940%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:43.949 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:43 smithi001 ceph-mon[107399]: pgmap v1672: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 10850/201192 objects degraded (5.393%); 5916/201192 objects misplaced (2.940%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:44.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:44.042 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:44.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:44.072 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:44.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:43 smithi012 ceph-mon[101180]: pgmap v1672: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 10850/201192 objects degraded (5.393%); 5916/201192 objects misplaced (2.940%); 20 MiB/s, 320 objects/s recovering 2024-09-14T04:31:44.099 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:44.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:44.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:44.301 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.338 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:44.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:44.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:44.480 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:44.582 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:44.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:44.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:44.758 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:44.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:44.888 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 10850/201192 objects degraded (5.393%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:44.888 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 10850/201192 objects degraded (5.393%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:44.899 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:44.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:44.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:45.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:45.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 10850/201192 objects degraded (5.393%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:45.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:45.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:45.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:45.310 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:45.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:45.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:45.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:45.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:45.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:45.723 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:45.825 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:45.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:45.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:45.986 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:45.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:46.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:45 smithi012 ceph-mon[101180]: pgmap v1673: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 9736/191904 objects degraded (5.073%); 5645/191904 objects misplaced (2.942%); 26 MiB/s, 419 objects/s recovering 2024-09-14T04:31:46.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.129 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:46.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:46.159 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:45 smithi001 ceph-mon[112165]: pgmap v1673: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 9736/191904 objects degraded (5.073%); 5645/191904 objects misplaced (2.942%); 26 MiB/s, 419 objects/s recovering 2024-09-14T04:31:46.159 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:45 smithi001 ceph-mon[107399]: pgmap v1673: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 9736/191904 objects degraded (5.073%); 5645/191904 objects misplaced (2.942%); 26 MiB/s, 419 objects/s recovering 2024-09-14T04:31:46.230 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:46.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:46.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:46.405 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:46.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:46.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:46.553 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:46.655 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:46.681 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:46.780 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:31:46.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:46.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:46.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:46.954 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:46.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3370546455' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3370546455' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:46.972 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:46] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.51.0" 2024-09-14T04:31:47.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:47.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3370546455' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:47.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:31:47.103 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:47.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:47.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:47.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:47.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:47.479 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:47.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:47.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:47.652 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:47.754 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:47.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:47.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:47.947 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:47.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:47.957 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:31:48.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:48.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.085 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:48.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:48.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:47 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:31:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:47 smithi012 ceph-mon[101180]: osdmap e528: 8 total, 8 up, 6 in 2024-09-14T04:31:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:47 smithi012 ceph-mon[101180]: pgmap v1675: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 9751/187704 objects degraded (5.195%); 5361/187704 objects misplaced (2.856%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:48.098 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:31:48.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[107399]: osdmap e528: 8 total, 8 up, 6 in 2024-09-14T04:31:48.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[107399]: pgmap v1675: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 9751/187704 objects degraded (5.195%); 5361/187704 objects misplaced (2.856%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:48.099 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:31:48.099 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[112165]: osdmap e528: 8 total, 8 up, 6 in 2024-09-14T04:31:48.099 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:47 smithi001 ceph-mon[112165]: pgmap v1675: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 9751/187704 objects degraded (5.195%); 5361/187704 objects misplaced (2.856%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:48.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:48.231 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:48.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:48.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:48.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:48.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:48.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.526 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:48.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:48.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:48.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:48.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:48.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:48.914 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:48.945 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:48.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:49.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:49.076 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:49.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:49.214 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:49.316 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:49.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:49.449 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:49.481 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:49.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:49.620 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:49.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 9751/187704 objects degraded (5.195%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:49.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 9751/187704 objects degraded (5.195%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:49.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:49.753 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:49.849 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 9751/187704 objects degraded (5.195%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:49.854 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:49.903 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:49.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:50.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:50.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:50.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:50.189 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:50.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:50.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:50.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:50 smithi001 ceph-mon[107399]: pgmap v1676: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 9751/186870 objects degraded (5.218%); 5361/186870 objects misplaced (2.869%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:50.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:50.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:50 smithi001 ceph-mon[112165]: pgmap v1676: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 9751/186870 objects degraded (5.218%); 5361/186870 objects misplaced (2.869%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:50.377 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:50.439 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:50.471 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:50.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:50 smithi012 ceph-mon[101180]: pgmap v1676: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 9751/186870 objects degraded (5.218%); 5361/186870 objects misplaced (2.869%); 20 MiB/s, 324 objects/s recovering 2024-09-14T04:31:50.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:50.605 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:50.706 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:50.737 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:50.839 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:50.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:50.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:50.974 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:51.004 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:51.106 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:51.132 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:51.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:51.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:51.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:51.417 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:51.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:51.553 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:51.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:51.686 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:31:51.686 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:31:51.686 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:31:51.686 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:31:51.687 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:31:51.687 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:31:51.687 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:31:51.687 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:31:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:51.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:51.712 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3834212619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:51.713 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3834212619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:51.713 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:31:51.713 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:31:51.752 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:31:51.752 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:31:51.780 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:31:51.780 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:31:51.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:51.807 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:31:51.807 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:31:51.827 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:51.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:51.968 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:51.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:52.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:52.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3834212619' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:31:52.111 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:52.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.257 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:52.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:52.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:52.397 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:52.498 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:52.529 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:52.631 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:52.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:52.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:52.800 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:52.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:52.942 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:52.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:52.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:52 smithi001 ceph-mon[107399]: pgmap v1677: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 887 op/s; 8685/176184 objects degraded (4.930%); 4722/176184 objects misplaced (2.680%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:52.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:52 smithi001 ceph-mon[112165]: pgmap v1677: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 887 op/s; 8685/176184 objects degraded (4.930%); 4722/176184 objects misplaced (2.680%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:53.044 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:53.073 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:53.091 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:53.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:52 smithi012 ceph-mon[101180]: pgmap v1677: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 887 op/s; 8685/176184 objects degraded (4.930%); 4722/176184 objects misplaced (2.680%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:53.175 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:53.209 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:53.311 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:53.347 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:53.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.488 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:53.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:53.590 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.634 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:53.735 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:53.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.766 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:53.867 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:53.904 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:53.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:53.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:54.007 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:54.041 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:54.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:54.185 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:54.192 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:31:54.192 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:31:54.286 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:54.316 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:54.418 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:54.450 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:54.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:54.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:54.592 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:54.652 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:31:54.690 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:31:54.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:54.769 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:54.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:54.871 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:54.908 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:54.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[107399]: pgmap v1678: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 8685/174432 objects degraded (4.979%); 4722/174432 objects misplaced (2.707%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/700358760' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 8685/174432 objects degraded (4.979%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[112165]: pgmap v1678: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 8685/174432 objects degraded (4.979%); 4722/174432 objects misplaced (2.707%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/700358760' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 8685/174432 objects degraded (4.979%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:55.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:55.049 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:55.086 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:31:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:54 smithi012 ceph-mon[101180]: pgmap v1678: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 8685/174432 objects degraded (4.979%); 4722/174432 objects misplaced (2.707%); 21 MiB/s, 335 objects/s recovering 2024-09-14T04:31:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/700358760' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:54 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:31:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 8685/174432 objects degraded (4.979%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:55.151 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:55.194 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:55.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:55.336 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:55.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:31:55.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:31:55.579 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:55.607 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:55.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:55.709 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:55.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:55.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:55.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[112165]: osdmap e529: 8 total, 8 up, 6 in 2024-09-14T04:31:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:55.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:55 smithi001 ceph-mon[107399]: osdmap e529: 8 total, 8 up, 6 in 2024-09-14T04:31:55.747 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:55.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:55.849 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:55.880 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:55.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:55.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:56.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:55 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:31:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:31:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:55 smithi012 ceph-mon[101180]: osdmap e529: 8 total, 8 up, 6 in 2024-09-14T04:31:56.130 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:56.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:56.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:56.304 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:56.406 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:56.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:56.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:56.579 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:56.681 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:56.711 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:56.813 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:56.843 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:56.944 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:56.970 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:56.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:56 smithi001 ceph-mon[112165]: pgmap v1680: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 7497/164619 objects degraded (4.554%); 4351/164619 objects misplaced (2.643%); 23 MiB/s, 368 objects/s recovering 2024-09-14T04:31:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:56 smithi001 ceph-mon[107399]: pgmap v1680: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 7497/164619 objects degraded (4.554%); 4351/164619 objects misplaced (2.643%); 23 MiB/s, 368 objects/s recovering 2024-09-14T04:31:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:31:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:31:56] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.51.0" 2024-09-14T04:31:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:56.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:57.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:56 smithi012 ceph-mon[101180]: pgmap v1680: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 7497/164619 objects degraded (4.554%); 4351/164619 objects misplaced (2.643%); 23 MiB/s, 368 objects/s recovering 2024-09-14T04:31:57.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:57.198 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:57.227 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:57.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:57.372 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:57.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:57.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:57.513 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:57.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:57.655 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:57.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:57.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:57.887 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:57.918 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:57.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:58.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:58.051 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:58.153 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:58.191 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:58.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:58.292 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:31:58.320 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:31:58.422 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:58.453 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:58.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:31:58.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.595 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:58.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:31:58.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:58.735 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:58.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:58.837 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:58.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:58.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:58.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:58 smithi001 ceph-mon[112165]: pgmap v1681: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 909 op/s; 7497/160416 objects degraded (4.673%); 3979/160416 objects misplaced (2.480%); 23 MiB/s, 363 objects/s recovering 2024-09-14T04:31:58.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:58 smithi001 ceph-mon[107399]: pgmap v1681: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 909 op/s; 7497/160416 objects degraded (4.673%); 3979/160416 objects misplaced (2.480%); 23 MiB/s, 363 objects/s recovering 2024-09-14T04:31:58.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:58.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:59.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:58 smithi012 ceph-mon[101180]: pgmap v1681: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 909 op/s; 7497/160416 objects degraded (4.673%); 3979/160416 objects misplaced (2.480%); 23 MiB/s, 363 objects/s recovering 2024-09-14T04:31:59.098 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:59.121 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:59.223 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:31:59.246 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:31:59.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:59.370 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:59.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:31:59.494 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:31:59.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:31:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.619 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:31:59.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:31:59.720 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:31:59.749 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:59.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:31:59.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:31:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:31:59.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:31:59.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:31:59.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:59 smithi001 ceph-mon[112165]: osdmap e530: 8 total, 8 up, 6 in 2024-09-14T04:31:59.916 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:31:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 7497/160416 objects degraded (4.673%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:59.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:59 smithi001 ceph-mon[107399]: osdmap e530: 8 total, 8 up, 6 in 2024-09-14T04:31:59.916 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:31:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 7497/160416 objects degraded (4.673%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:31:59.929 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:59.929 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:31:59.986 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:00.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:00.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:59 smithi012 ceph-mon[101180]: osdmap e530: 8 total, 8 up, 6 in 2024-09-14T04:32:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:31:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 7497/160416 objects degraded (4.673%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:00.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:00.169 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:00.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:00.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:00.312 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:00.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:00.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:00.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:00.589 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:00.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:00.726 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:00.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:00.858 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:00.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:00 smithi001 ceph-mon[107399]: pgmap v1683: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 7497/159582 objects degraded (4.698%); 3979/159582 objects misplaced (2.493%); 15 MiB/s, 241 objects/s recovering 2024-09-14T04:32:00.907 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:00 smithi001 ceph-mon[107399]: osdmap e531: 8 total, 8 up, 6 in 2024-09-14T04:32:00.908 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:00 smithi001 ceph-mon[112165]: pgmap v1683: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 7497/159582 objects degraded (4.698%); 3979/159582 objects misplaced (2.493%); 15 MiB/s, 241 objects/s recovering 2024-09-14T04:32:00.908 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:00 smithi001 ceph-mon[112165]: osdmap e531: 8 total, 8 up, 6 in 2024-09-14T04:32:00.960 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:00.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:00.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:01.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:01.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:00 smithi012 ceph-mon[101180]: pgmap v1683: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 7497/159582 objects degraded (4.698%); 3979/159582 objects misplaced (2.493%); 15 MiB/s, 241 objects/s recovering 2024-09-14T04:32:01.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:00 smithi012 ceph-mon[101180]: osdmap e531: 8 total, 8 up, 6 in 2024-09-14T04:32:01.114 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:01.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:01.238 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:01.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:01.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:01.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:01.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:01.488 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:01.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:01.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.613 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:01.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:01.678 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:32:01.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:01.745 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:01.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:01.847 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:01.882 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:01.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:01.966 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:32:01.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:01 smithi001 ceph-mon[107399]: osdmap e532: 8 total, 8 up, 6 in 2024-09-14T04:32:01.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:01 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:32:01.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:01 smithi001 ceph-mon[112165]: osdmap e532: 8 total, 8 up, 6 in 2024-09-14T04:32:01.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:01 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:32:01.983 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.020 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:02.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:01 smithi012 ceph-mon[101180]: osdmap e532: 8 total, 8 up, 6 in 2024-09-14T04:32:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:01 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:32:02.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:02.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:02.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:02.276 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:02.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:02.400 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:02.502 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:02.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.524 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:02.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:02.648 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:02.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:02.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:02.890 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:02.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:02.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:02.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:02.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[107399]: pgmap v1686: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 6231/149583 objects degraded (4.166%); 3354/149583 objects misplaced (2.242%); 22 MiB/s, 377 objects/s recovering 2024-09-14T04:32:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:32:02.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[107399]: osdmap e533: 8 total, 8 up, 6 in 2024-09-14T04:32:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[112165]: pgmap v1686: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 6231/149583 objects degraded (4.166%); 3354/149583 objects misplaced (2.242%); 22 MiB/s, 377 objects/s recovering 2024-09-14T04:32:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:32:02.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:02 smithi001 ceph-mon[112165]: osdmap e533: 8 total, 8 up, 6 in 2024-09-14T04:32:03.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:03.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:03.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:02 smithi012 ceph-mon[101180]: pgmap v1686: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 6231/149583 objects degraded (4.166%); 3354/149583 objects misplaced (2.242%); 22 MiB/s, 377 objects/s recovering 2024-09-14T04:32:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/503354752' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:32:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:02 smithi012 ceph-mon[101180]: osdmap e533: 8 total, 8 up, 6 in 2024-09-14T04:32:03.162 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:03.192 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:03.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:03.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:03.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:03.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:03.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:03.598 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:03.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:03.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:03.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:03.845 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:03.879 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:03.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:03.897 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:03 smithi001 ceph-mon[112165]: osdmap e534: 8 total, 8 up, 6 in 2024-09-14T04:32:03.897 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:03 smithi001 ceph-mon[107399]: osdmap e534: 8 total, 8 up, 6 in 2024-09-14T04:32:03.981 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:04.014 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:03 smithi012 ceph-mon[101180]: osdmap e534: 8 total, 8 up, 6 in 2024-09-14T04:32:04.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:04.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:04.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:04.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:04.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:04.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:04.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:32:04.422 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:04.524 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:04.558 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:04.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:04.699 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:04.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:04.801 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:04.832 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:04.934 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:04.964 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:04.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:04.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[112165]: 14.1 scrub starts 2024-09-14T04:32:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[112165]: pgmap v1689: 119 pgs: 2 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 5776/148356 objects degraded (3.893%); 3354/148356 objects misplaced (2.261%); 29 MiB/s, 586 objects/s recovering 2024-09-14T04:32:04.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 5776/148356 objects degraded (3.893%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[107399]: 14.1 scrub starts 2024-09-14T04:32:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[107399]: pgmap v1689: 119 pgs: 2 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 5776/148356 objects degraded (3.893%); 3354/148356 objects misplaced (2.261%); 29 MiB/s, 586 objects/s recovering 2024-09-14T04:32:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 5776/148356 objects degraded (3.893%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:05.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:04 smithi012 ceph-mon[101180]: 14.1 scrub starts 2024-09-14T04:32:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:04 smithi012 ceph-mon[101180]: pgmap v1689: 119 pgs: 2 peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 935 op/s; 5776/148356 objects degraded (3.893%); 3354/148356 objects misplaced (2.261%); 29 MiB/s, 586 objects/s recovering 2024-09-14T04:32:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 5776/148356 objects degraded (3.893%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:05.108 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:05.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:05.237 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:05.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:05.383 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:05.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:05.520 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:05.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:05.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:05.661 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:05.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:05.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:05.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:05.912 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:05.913 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:05.913 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:05.973 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:32:05.974 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:32:05.978 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:05.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:06.080 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:06.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:06.113 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:06.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.255 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:06.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:06.356 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:06.385 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:06.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:06.521 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:06.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:06.630 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:06.663 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:06.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:06.682 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:32:06.683 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:32:06.713 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:32:06.714 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:32:06.743 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:32:06.744 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:32:06.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:06.771 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:32:06.772 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:32:06.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:06 smithi001 ceph-mon[107399]: pgmap v1690: 119 pgs: 1 active+recovering+remapped, 2 peering, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 4850/140895 objects degraded (3.442%); 3222/140895 objects misplaced (2.287%); 31 MiB/s, 570 objects/s recovering 2024-09-14T04:32:06.772 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1729702573' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:06.773 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:06 smithi001 ceph-mon[112165]: pgmap v1690: 119 pgs: 1 active+recovering+remapped, 2 peering, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 4850/140895 objects degraded (3.442%); 3222/140895 objects misplaced (2.287%); 31 MiB/s, 570 objects/s recovering 2024-09-14T04:32:06.773 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1729702573' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:06.773 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:06] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.51.0" 2024-09-14T04:32:06.796 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:32:06.796 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:32:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.809 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:06.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:06.833 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:32:06.833 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:32:06.869 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:32:06.869 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:32:06.899 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:32:06.900 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:32:06.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:06.946 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:06.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:07.047 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:07.079 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:06 smithi012 ceph-mon[101180]: pgmap v1690: 119 pgs: 1 active+recovering+remapped, 2 peering, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 4850/140895 objects degraded (3.442%); 3222/140895 objects misplaced (2.287%); 31 MiB/s, 570 objects/s recovering 2024-09-14T04:32:07.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1729702573' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:07.181 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:07.215 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:32:07.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:07.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:07.379 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:07.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:07.523 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:07.625 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:07.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:07.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:07.785 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:07.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:07.926 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:07.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:08.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:08.063 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:08.165 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:08.193 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:08.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:08.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:08.335 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:08.437 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:08.467 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:08.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:08.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.611 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:08.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:08.712 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:08.747 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:08.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:08.896 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:08.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:08.965 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:08 smithi001 ceph-mon[107399]: pgmap v1691: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 4850/136875 objects degraded (3.543%); 3222/136875 objects misplaced (2.354%); 14 MiB/s, 299 objects/s recovering 2024-09-14T04:32:08.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:08 smithi001 ceph-mon[112165]: pgmap v1691: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 4850/136875 objects degraded (3.543%); 3222/136875 objects misplaced (2.354%); 14 MiB/s, 299 objects/s recovering 2024-09-14T04:32:08.998 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:09.031 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:09.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:08 smithi012 ceph-mon[101180]: pgmap v1691: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 714 op/s; 4850/136875 objects degraded (3.543%); 3222/136875 objects misplaced (2.354%); 14 MiB/s, 299 objects/s recovering 2024-09-14T04:32:09.133 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:09.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:09.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.311 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:09.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:09.412 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:09.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:09.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:09.573 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:09.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:09.715 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:09.817 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:09.847 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:09.949 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:09.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:09 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4850/136875 objects degraded (3.543%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:09.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:09 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4850/136875 objects degraded (3.543%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:09.981 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:09.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:10.064 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:10.083 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:10.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:09 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4850/136875 objects degraded (3.543%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.135 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:10.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:10.236 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:10.268 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:10.370 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:10.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:10.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:10.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:10.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:10.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:10.672 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:10.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:10.814 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:10.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:10.898 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:10 smithi001 ceph-mon[107399]: pgmap v1692: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4850/135840 objects degraded (3.570%); 3081/135840 objects misplaced (2.268%); 10 MiB/s, 206 objects/s recovering 2024-09-14T04:32:10.898 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:10 smithi001 ceph-mon[112165]: pgmap v1692: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4850/135840 objects degraded (3.570%); 3081/135840 objects misplaced (2.268%); 10 MiB/s, 206 objects/s recovering 2024-09-14T04:32:10.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:10.962 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:10.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:10.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:11.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:11.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:11.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:10 smithi012 ceph-mon[101180]: pgmap v1692: 119 pgs: 1 active+recovering+remapped, 9 active+recovery_wait+degraded, 4 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4850/135840 objects degraded (3.570%); 3081/135840 objects misplaced (2.268%); 10 MiB/s, 206 objects/s recovering 2024-09-14T04:32:11.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:11.231 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:11.334 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:11.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:11.495 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:11.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.504 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:11.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:11.605 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:11.637 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:11.739 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:11.780 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:11.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:11.913 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:11.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:11.965 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:11 smithi001 ceph-mon[107399]: pgmap v1693: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 4621/122316 objects degraded (3.778%); 3081/122316 objects misplaced (2.519%); 9.6 MiB/s, 189 objects/s recovering 2024-09-14T04:32:11.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:11 smithi001 ceph-mon[112165]: pgmap v1693: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 4621/122316 objects degraded (3.778%); 3081/122316 objects misplaced (2.519%); 9.6 MiB/s, 189 objects/s recovering 2024-09-14T04:32:12.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:12.054 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:12.090 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:32:12.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:11 smithi012 ceph-mon[101180]: pgmap v1693: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 4621/122316 objects degraded (3.778%); 3081/122316 objects misplaced (2.519%); 9.6 MiB/s, 189 objects/s recovering 2024-09-14T04:32:12.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:12.200 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:12.302 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:12.348 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:12.450 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.490 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:12.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:12.591 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:12.622 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:12.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:12.736 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:32:12.763 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:12.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:12.902 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:12.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:12.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1415015319' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:12.993 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:12.994 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1415015319' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:12.994 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:13.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:13.043 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1415015319' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:32:13.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:13.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:13.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:13.304 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:13.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:13.449 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:13.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.578 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:13.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:13.679 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:13.707 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:13.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.841 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:13.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:13.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:13.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:13.986 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:13.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:14.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:13 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:13 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:32:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:13 smithi012 ceph-mon[101180]: osdmap e535: 8 total, 8 up, 6 in 2024-09-14T04:32:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:13 smithi012 ceph-mon[101180]: pgmap v1695: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 4621/120150 objects degraded (3.846%); 3081/120150 objects misplaced (2.564%); 8.5 MiB/s, 142 objects/s recovering 2024-09-14T04:32:14.124 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:14.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[107399]: osdmap e535: 8 total, 8 up, 6 in 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[107399]: pgmap v1695: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 4621/120150 objects degraded (3.846%); 3081/120150 objects misplaced (2.564%); 8.5 MiB/s, 142 objects/s recovering 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[112165]: osdmap e535: 8 total, 8 up, 6 in 2024-09-14T04:32:14.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:13 smithi001 ceph-mon[112165]: pgmap v1695: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 4621/120150 objects degraded (3.846%); 3081/120150 objects misplaced (2.564%); 8.5 MiB/s, 142 objects/s recovering 2024-09-14T04:32:14.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:14.265 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:14.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:14.406 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:14.508 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.545 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:14.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:14.646 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:14.685 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:14.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:14.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:14.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:14.901 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:14 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/120150 objects degraded (3.846%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:14.901 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:14 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/120150 objects degraded (3.846%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:14.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:14.955 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:14.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:15.056 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:15.086 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:14 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/120150 objects degraded (3.846%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:15.188 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:15.221 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:15.323 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.360 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:15.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:15.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:15.502 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:15.604 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.644 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:15.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:15.745 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:15.777 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:15.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:15.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:15.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:15.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:15 smithi001 ceph-mon[112165]: pgmap v1696: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 4621/108726 objects degraded (4.250%); 2938/108726 objects misplaced (2.702%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:15.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:15 smithi001 ceph-mon[107399]: pgmap v1696: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 4621/108726 objects degraded (4.250%); 2938/108726 objects misplaced (2.702%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:16.018 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:16.051 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:16.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:15 smithi012 ceph-mon[101180]: pgmap v1696: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 4621/108726 objects degraded (4.250%); 2938/108726 objects misplaced (2.702%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:16.152 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:16.191 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:16.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:16.319 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:16.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:16.459 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:16.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:16.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:16.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:16.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:16.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:16.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:16.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:16.881 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:16.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:16.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:16] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.51.0" 2024-09-14T04:32:16.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:17.011 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:17.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.144 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:17.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:17.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:17.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:17.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:17.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:17.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:17.550 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:17.652 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:17.688 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:17.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:17.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:17 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.848 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:17.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:17.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:17.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:17 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:17.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:17 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:17.995 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:17.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:18.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:18.126 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:18.228 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:18.260 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:18.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:18.387 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:18.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.536 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:18.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:18.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:18.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:18.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:18.826 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:18.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:18 smithi012 ceph-mon[101180]: pgmap v1697: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/103704 objects degraded (4.456%); 2938/103704 objects misplaced (2.833%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:18.852 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:18 smithi001 ceph-mon[112165]: pgmap v1697: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/103704 objects degraded (4.456%); 2938/103704 objects misplaced (2.833%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:18.852 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:18 smithi001 ceph-mon[107399]: pgmap v1697: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/103704 objects degraded (4.456%); 2938/103704 objects misplaced (2.833%); 1.9 MiB/s, 51 objects/s recovering 2024-09-14T04:32:18.928 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:18.965 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:18.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:19.067 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.099 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:19.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:19.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:19.355 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:19.386 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:19.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:19.519 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:19.621 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:19.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:19.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:19 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/103704 objects degraded (4.456%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:19.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:19 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/103704 objects degraded (4.456%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:19.752 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:19.789 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:19 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/103704 objects degraded (4.456%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:19.891 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:19.921 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:19.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:20.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.071 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:20.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:20.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:20.221 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:20.322 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:20.374 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:20.420 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:32:20.420 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:32:20.420 INFO:teuthology.orchestra.run.smithi001.stdout: "complaint_time": 30, 2024-09-14T04:32:20.420 INFO:teuthology.orchestra.run.smithi001.stdout: "num_blocked_ops": 0 2024-09-14T04:32:20.420 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:32:20.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:20.540 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:20.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:20.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.678 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:20.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:20.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:20.814 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:20 smithi012 ceph-mon[101180]: pgmap v1698: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/102657 objects degraded (4.501%); 2790/102657 objects misplaced (2.718%); 1.4 MiB/s, 51 objects/s recovering 2024-09-14T04:32:20.815 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:20.818 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:20.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:20.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:20.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:20 smithi001 ceph-mon[112165]: pgmap v1698: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/102657 objects degraded (4.501%); 2790/102657 objects misplaced (2.718%); 1.4 MiB/s, 51 objects/s recovering 2024-09-14T04:32:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:20 smithi001 ceph-mon[107399]: pgmap v1698: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 4621/102657 objects degraded (4.501%); 2790/102657 objects misplaced (2.718%); 1.4 MiB/s, 51 objects/s recovering 2024-09-14T04:32:20.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:20.972 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:20.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:21.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:21.101 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:21.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:32:21.169 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:21.203 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:21.272 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:21.374 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:21.407 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:21.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:32:21.510 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:21.543 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:32:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.586 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:21.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:21.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:21.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:21 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1202096935' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:21 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:21 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1202096935' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:21 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.741 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:21.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:21 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1202096935' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:21 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:32:21.887 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:21.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:21.989 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:22.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.029 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:22.130 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:22.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:22.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:22.309 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:22.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:22.457 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:22.560 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:22.590 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:22.691 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:22.726 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:32:22.729 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:22.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:22 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:32:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:22 smithi012 ceph-mon[101180]: osdmap e536: 8 total, 8 up, 6 in 2024-09-14T04:32:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:22 smithi012 ceph-mon[101180]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 4621/90021 objects degraded (5.133%); 2790/90021 objects misplaced (3.099%); 0 B/s, 32 objects/s recovering 2024-09-14T04:32:22.856 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:32:22.857 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[107399]: osdmap e536: 8 total, 8 up, 6 in 2024-09-14T04:32:22.857 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[107399]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 4621/90021 objects degraded (5.133%); 2790/90021 objects misplaced (3.099%); 0 B/s, 32 objects/s recovering 2024-09-14T04:32:22.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:32:22.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[112165]: osdmap e536: 8 total, 8 up, 6 in 2024-09-14T04:32:22.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:22 smithi001 ceph-mon[112165]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 4621/90021 objects degraded (5.133%); 2790/90021 objects misplaced (3.099%); 0 B/s, 32 objects/s recovering 2024-09-14T04:32:22.862 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:22.964 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:22.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:22.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:23.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:23.130 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:23.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:23.271 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:23.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:23.412 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:23.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:23.552 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:23.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:23.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:23.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:23.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:23.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.835 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:23.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:23.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:23.971 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:23.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:24.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:24.106 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:24.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:24.247 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:24.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:24.392 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:24.495 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:24.522 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:24.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:24.624 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:24.657 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:24.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:24.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:24.817 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:24.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:24.958 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:24.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:24.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:24 smithi001 ceph-mon[107399]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/88200 objects degraded (5.239%); 2790/88200 objects misplaced (3.163%); 0 B/s, 29 objects/s recovering 2024-09-14T04:32:24.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:24 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/88200 objects degraded (5.239%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:24 smithi001 ceph-mon[112165]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/88200 objects degraded (5.239%); 2790/88200 objects misplaced (3.163%); 0 B/s, 29 objects/s recovering 2024-09-14T04:32:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:24 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/88200 objects degraded (5.239%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:25.059 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:25.094 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:25.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:24 smithi012 ceph-mon[101180]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/88200 objects degraded (5.239%); 2790/88200 objects misplaced (3.163%); 0 B/s, 29 objects/s recovering 2024-09-14T04:32:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:24 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/88200 objects degraded (5.239%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:25.196 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:25.229 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:25.331 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:25.364 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:25.466 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:25.496 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:25.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:25.624 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:25.726 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:25.764 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:25.808 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:25.809 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:25.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:25.911 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:25.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:26.013 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:26.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.044 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:26.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:26.145 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.188 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:26.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:26.289 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:26.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.329 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:26.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:26.353 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:26.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:26.432 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:32:26.432 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:32:26.463 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:32:26.463 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:32:26.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:26.485 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:32:26.486 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:32:26.515 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:32:26.515 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:32:26.546 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:32:26.546 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:32:26.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:26.582 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:32:26.583 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:32:26.583 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.583 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.584 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[107399]: pgmap v1702: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/77820 objects degraded (5.938%); 2655/77820 objects misplaced (3.412%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:26.584 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.584 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.584 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-mon[112165]: pgmap v1702: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/77820 objects degraded (5.938%); 2655/77820 objects misplaced (3.412%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:26.597 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:26 smithi012 ceph-mon[101180]: pgmap v1702: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/77820 objects degraded (5.938%); 2655/77820 objects misplaced (3.412%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:26.608 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:26.613 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:32:26.613 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:32:26.641 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:32:26.641 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:32:26.710 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:26.759 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:26.861 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:26.890 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:26.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:26] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.51.0" 2024-09-14T04:32:26.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:27.031 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:27.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:27.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.171 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:27.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:27.272 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:27.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:27.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:27.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:27.446 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:27.548 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:27.581 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3339393724' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:27.581 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3339393724' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:27.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:27.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3339393724' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:27.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:27.717 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:27.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:27.860 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:27.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:27.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:28.002 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:28.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:28.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.146 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:28.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:28.247 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:28.277 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:28.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:28.379 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:28.409 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:28.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:28.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:28.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:28.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:28.652 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:28 smithi001 ceph-mon[112165]: pgmap v1703: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/73197 objects degraded (6.313%); 2655/73197 objects misplaced (3.627%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:28.653 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:28 smithi001 ceph-mon[107399]: pgmap v1703: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/73197 objects degraded (6.313%); 2655/73197 objects misplaced (3.627%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:28.681 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:28.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:28.828 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:28.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:28 smithi012 ceph-mon[101180]: pgmap v1703: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/73197 objects degraded (6.313%); 2655/73197 objects misplaced (3.627%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:28.930 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:28.963 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:28.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:29.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:29.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:29.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.239 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:29.340 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:29.381 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:29.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.525 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:29.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:29.630 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:29 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/73197 objects degraded (6.313%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:29.630 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:29 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/73197 objects degraded (6.313%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:29.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:29.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.812 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:29.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:29 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/73197 objects degraded (6.313%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:29.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:29.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:29.937 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:29.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:30.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:30.082 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:30.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:30.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:30.219 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:30.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:30.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.369 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:30.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:30.463 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:30 smithi001 ceph-mon[112165]: pgmap v1704: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 4621/72174 objects degraded (6.403%); 2507/72174 objects misplaced (3.474%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:30.463 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:30 smithi001 ceph-mon[107399]: pgmap v1704: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 4621/72174 objects degraded (6.403%); 2507/72174 objects misplaced (3.474%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:30.471 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:30.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:30.611 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:30.642 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:30.744 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:30.774 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:30.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:30 smithi012 ceph-mon[101180]: pgmap v1704: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 4621/72174 objects degraded (6.403%); 2507/72174 objects misplaced (3.474%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:30.876 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:30.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:30.909 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:30.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:31.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:31.048 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:31.150 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:31.181 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:31.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:31.317 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:31.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:31.461 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:31.563 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:31.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:31.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:31.664 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:32:31.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:31.731 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:31.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:31.866 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:31.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:31.968 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:32.022 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:32.126 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.165 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:32.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:32.268 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:32.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:32.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:32.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:32.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:32.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:32.552 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:32:32.552 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:32:32.577 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:32.680 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:32.699 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:32:32.701 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:32:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:32.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:32.821 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:32.861 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:32.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:32.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:32.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[107399]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/58992 objects degraded (7.833%); 2507/58992 objects misplaced (4.250%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:32.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:32:32.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:32:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[112165]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/58992 objects degraded (7.833%); 2507/58992 objects misplaced (4.250%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:32:32.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:32 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:32:33.010 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:32 smithi012 ceph-mon[101180]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4621/58992 objects degraded (7.833%); 2507/58992 objects misplaced (4.250%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:32:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:32 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:32:33.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:33.148 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:33.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:33.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:33.278 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:33.379 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:33.417 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:33.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.557 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:33.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:33.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:33.692 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:33.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:33.829 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:33.931 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:33.961 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:33.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:34.063 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.096 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:34.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:33 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:32:34.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:33 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:32:34.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:33 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:32:34.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:33 smithi012 ceph-mon[101180]: osdmap e537: 8 total, 8 up, 6 in 2024-09-14T04:32:34.197 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:34.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[112165]: osdmap e537: 8 total, 8 up, 6 in 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/297420245' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3731214527' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:32:34.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:33 smithi001 ceph-mon[107399]: osdmap e537: 8 total, 8 up, 6 in 2024-09-14T04:32:34.228 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:34.330 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:34.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:34.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:34.505 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:34.607 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.649 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:34.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:34.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:34.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:32:34.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:34.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:34.920 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:34.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:34.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:35.023 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:35.052 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:34 smithi012 ceph-mon[101180]: 14.1 scrub starts 2024-09-14T04:32:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:34 smithi012 ceph-mon[101180]: 14.1 scrub ok 2024-09-14T04:32:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:34 smithi012 ceph-mon[101180]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/57153 objects degraded (8.085%); 2507/57153 objects misplaced (4.386%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:35.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:34 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/57153 objects degraded (8.085%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:35.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:35.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[112165]: 14.1 scrub starts 2024-09-14T04:32:35.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[112165]: 14.1 scrub ok 2024-09-14T04:32:35.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[112165]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/57153 objects degraded (8.085%); 2507/57153 objects misplaced (4.386%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:35.170 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/57153 objects degraded (8.085%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:35.170 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[107399]: 14.1 scrub starts 2024-09-14T04:32:35.171 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[107399]: 14.1 scrub ok 2024-09-14T04:32:35.171 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[107399]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 4621/57153 objects degraded (8.085%); 2507/57153 objects misplaced (4.386%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:35.171 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:34 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/57153 objects degraded (8.085%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:35.173 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:32:35.180 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:35.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:35.305 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:35.335 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:35.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.477 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:35.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:35.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:35.618 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:35.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:35.750 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:32:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.761 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:35.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:35.862 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:35.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:35.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:35.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:36.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:36.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:35 smithi012 ceph-mon[101180]: pgmap v1708: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 957 MiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/45972 objects degraded (10.052%); 2366/45972 objects misplaced (5.147%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:36.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:36.176 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:36.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[112165]: pgmap v1708: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 957 MiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/45972 objects degraded (10.052%); 2366/45972 objects misplaced (5.147%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:36.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:35 smithi001 ceph-mon[107399]: pgmap v1708: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 957 MiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 4621/45972 objects degraded (10.052%); 2366/45972 objects misplaced (5.147%); 0 B/s, 28 objects/s recovering 2024-09-14T04:32:36.278 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:36.309 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:36.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:36.451 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:36.552 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:36.585 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:36.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:36.736 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:36.838 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:36.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:36.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:36.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:36] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:32:36.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:37.010 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:37.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:37.156 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:37.210 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:32:37.258 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:37.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.298 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:37.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:37.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:37.443 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:37.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.583 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:37.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:37.684 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.732 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:37.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:37.833 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:37.865 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:37.967 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:37.996 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:37.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:38.011 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:32:38.011 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:32:38.086 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:38.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.132 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:38.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:38.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:38.277 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:38.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:38.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:38.445 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:32:38.446 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:32:38.475 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:32:38.475 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:32:38.500 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:32:38.501 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:32:38.531 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:32:38.531 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:32:38.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:38.565 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:32:38.565 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:32:38.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:38.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:38.596 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:32:38.596 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:32:38.625 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:32:38.625 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:32:38.655 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:32:38.656 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:32:38.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:38.709 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:38.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:32:38.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:38.869 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:38.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:38.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:38 smithi001 ceph-mon[107399]: pgmap v1709: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 852 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/40944 objects degraded (11.286%); 2366/40944 objects misplaced (5.779%) 2024-09-14T04:32:38.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:38 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4062163383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:38 smithi001 ceph-mon[112165]: pgmap v1709: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 852 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/40944 objects degraded (11.286%); 2366/40944 objects misplaced (5.779%) 2024-09-14T04:32:38.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:38 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4062163383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:38.971 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:39.004 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:38 smithi012 ceph-mon[101180]: pgmap v1709: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 852 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/40944 objects degraded (11.286%); 2366/40944 objects misplaced (5.779%) 2024-09-14T04:32:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:38 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4062163383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:32:39.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:39.151 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:39.178 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:32:39.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:39.299 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:39.401 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:39.436 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:39.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:39.584 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:39.686 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:39.720 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:39.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:39.859 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:39.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:39.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:39 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/40944 objects degraded (11.286%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:39 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/40944 objects degraded (11.286%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:39.997 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:39.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:39.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:40.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:39 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/40944 objects degraded (11.286%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:40.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:40.139 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:40.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:40.282 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:40.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:40.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:40.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:40.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.567 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:40.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:40.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:40.726 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:40.765 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:40 smithi012 ceph-mon[101180]: pgmap v1710: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 831 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/39936 objects degraded (11.571%); 2227/39936 objects misplaced (5.576%) 2024-09-14T04:32:40.829 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:40.859 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:40.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:40.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:40.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:40 smithi001 ceph-mon[107399]: pgmap v1710: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 831 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/39936 objects degraded (11.571%); 2227/39936 objects misplaced (5.576%) 2024-09-14T04:32:40.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:40 smithi001 ceph-mon[112165]: pgmap v1710: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 831 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 4621/39936 objects degraded (11.571%); 2227/39936 objects misplaced (5.576%) 2024-09-14T04:32:40.989 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:40.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:41.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:41.118 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:41.217 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:41.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:41.270 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:41.372 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:41.402 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:41.504 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:41.532 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:41.634 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:41.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:41.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:41.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:41.798 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:41.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:41.926 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:32:41.926 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2024-09-14T04:32:41.926 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2024-09-14T04:32:41.926 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2024-09-14T04:32:41.926 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:32:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:41.945 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:41.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:42.046 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:42.075 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:42.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:42.177 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:42.205 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:42.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:42.351 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:42.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:42.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:42.492 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:42.593 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:42.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:42.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:32:42.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:42.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:42.882 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:42.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:42.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:42.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:42 smithi001 ceph-mon[112165]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 571 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4621/27441 objects degraded (16.840%); 2227/27441 objects misplaced (8.116%) 2024-09-14T04:32:42.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:42 smithi001 ceph-mon[107399]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 571 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4621/27441 objects degraded (16.840%); 2227/27441 objects misplaced (8.116%) 2024-09-14T04:32:43.017 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:43.035 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:32:43.048 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:42 smithi012 ceph-mon[101180]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 571 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4621/27441 objects degraded (16.840%); 2227/27441 objects misplaced (8.116%) 2024-09-14T04:32:43.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:43.206 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:43.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.334 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:43.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:43.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:43.466 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:43.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:43.609 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:43.714 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:43.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:43.852 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:43.886 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:43.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:43.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:44.020 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:44.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:44.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:44.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:44.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:44.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:44.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:44.444 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:44.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:44.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:44.600 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:44.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:44.701 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:44.736 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:44.838 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:44.877 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:44.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:44.979 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:45.013 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:44 smithi012 ceph-mon[101180]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 532 MiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 4621/25599 objects degraded (18.051%); 2227/25599 objects misplaced (8.700%) 2024-09-14T04:32:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:44 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/25599 objects degraded (18.051%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:45.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.153 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:45.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:45.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:44 smithi001 ceph-mon[107399]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 532 MiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 4621/25599 objects degraded (18.051%); 2227/25599 objects misplaced (8.700%) 2024-09-14T04:32:45.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:44 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/25599 objects degraded (18.051%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:44 smithi001 ceph-mon[112165]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 532 MiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 4621/25599 objects degraded (18.051%); 2227/25599 objects misplaced (8.700%) 2024-09-14T04:32:45.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:44 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/25599 objects degraded (18.051%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:45.253 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:45.282 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:45.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:45.416 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:45.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:45.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:45.651 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:45.682 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:45.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:45.820 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:45.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:45.960 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:45.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:46.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:46.079 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:45 smithi012 ceph-mon[101180]: pgmap v1713: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 328 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 4621/15789 objects degraded (29.267%); 2072/15789 objects misplaced (13.123%) 2024-09-14T04:32:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.100 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:46.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:46.204 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:46.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:45 smithi001 ceph-mon[107399]: pgmap v1713: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 328 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 4621/15789 objects degraded (29.267%); 2072/15789 objects misplaced (13.123%) 2024-09-14T04:32:46.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:45 smithi001 ceph-mon[112165]: pgmap v1713: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 328 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 4621/15789 objects degraded (29.267%); 2072/15789 objects misplaced (13.123%) 2024-09-14T04:32:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.234 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:46.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:46.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:46.379 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:46.420 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-14T04:32:46.420 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49322 13.2 13:442a2ab9:::benchmark_data_smithi001_306460_object49321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.561442+0000", 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.853235767, 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4926849010000001, 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49322 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561442+0000", 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561442+0000", 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561437+0000", 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561483+0000", 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6275e-05 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561501+0000", 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7858000000000001e-05 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561508+0000", 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7519999999999996e-06 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561572+0000", 2024-09-14T04:32:46.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3502000000000003e-05 2024-09-14T04:32:46.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.561582+0000", 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0114e-05 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551272+0000", 2024-09-14T04:32:46.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.989689818 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551284+0000", 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2573e-05 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.046961+0000", 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4956762989999999 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.047065+0000", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104577 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.047185+0000", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119821 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.052143+0000", 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049581510000000001 2024-09-14T04:32:46.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.052198+0000", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4839000000000001e-05 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.052209+0000", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0841e-05 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.052270+0000", 2024-09-14T04:32:46.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0869000000000003e-05 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.054126+0000", 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001856573 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49334 13.2 13:425de8b2:::benchmark_data_smithi001_306460_object49333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.574778+0000", 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.83989959899998, 2024-09-14T04:32:46.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4809788680000002, 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49334 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574778+0000", 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574778+0000", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574776+0000", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574818+0000", 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1561999999999999e-05 2024-09-14T04:32:46.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574833+0000", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4780000000000001e-05 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574840+0000", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7279999999999995e-06 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574906+0000", 2024-09-14T04:32:46.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5914000000000006e-05 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.574916+0000", 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8099999999999992e-06 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551310+0000", 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97639357800000004 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551317+0000", 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9330000000000004e-06 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.050299+0000", 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4989821509999999 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.050389+0000", 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0563999999999996e-05 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.050495+0000", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106073 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.054262+0000", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037661539999999999 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.054270+0000", 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.303e-06 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.054311+0000", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1310000000000003e-05 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.054365+0000", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3381999999999998e-05 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.055757+0000", 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00139196 2024-09-14T04:32:46.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49335 13.2 13:4f4e395a:::benchmark_data_smithi001_306460_object49334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.575044+0000", 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.83963358099999, 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4886049469999998, 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.439 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49335 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575044+0000", 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575044+0000", 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575043+0000", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575107+0000", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3960000000000004e-05 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575116+0000", 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3190000000000001e-06 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575121+0000", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3379999999999998e-06 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575156+0000", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5287999999999998e-05 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.575163+0000", 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0299999999999996e-06 2024-09-14T04:32:46.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551340+0000", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97617728599999998 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551348+0000", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.055807+0000", 2024-09-14T04:32:46.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.504458914 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.055941+0000", 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133614 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.056100+0000", 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015885899999999999 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.062515+0000", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064154140000000004 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.062564+0000", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8742999999999997e-05 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.062615+0000", 2024-09-14T04:32:46.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1181999999999999e-05 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.062686+0000", 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0593999999999995e-05 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063649+0000", 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00096290600000000003 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.446 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49348 13.2 13:4d1dc6ce:::benchmark_data_smithi001_306460_object49347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.583620+0000", 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.831057767, 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4851650169999999, 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49348 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.447 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583620+0000", 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583620+0000", 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583618+0000", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583669+0000", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1010999999999997e-05 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583688+0000", 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8960000000000001e-05 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583696+0000", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0639999999999994e-06 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583755+0000", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8109999999999998e-05 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.583767+0000", 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2483000000000001e-05 2024-09-14T04:32:46.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551370+0000", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96760285199999996 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551377+0000", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8229999999999997e-06 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.057554+0000", 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.506177331 2024-09-14T04:32:46.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.057661+0000", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106699 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.057805+0000", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014453799999999999 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063669+0000", 2024-09-14T04:32:46.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058640660000000002 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063706+0000", 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7069e-05 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.066704+0000", 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029979799999999999 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.066737+0000", 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2877999999999999e-05 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.068785+0000", 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020473359999999999 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49366 13.2 13:498f622f:::benchmark_data_smithi001_306460_object49365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.596178+0000", 2024-09-14T04:32:46.455 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.81849949799999, 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4748084480000001, 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49366 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596178+0000", 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596178+0000", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596177+0000", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596250+0000", 2024-09-14T04:32:46.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3263000000000007e-05 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596264+0000", 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3732e-05 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596271+0000", 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3629999999999996e-06 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596339+0000", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7544000000000002e-05 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.596351+0000", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1989e-05 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551399+0000", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95504864599999995 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551407+0000", 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2819999999999999e-06 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.059147+0000", 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5077402360000001 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.059282+0000", 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135086 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.059478+0000", 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000196102 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063681+0000", 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004203119 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063727+0000", 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6245999999999997e-05 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.068871+0000", 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051431000000000003 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.068931+0000", 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0112999999999999e-05 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.070986+0000", 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020555880000000001 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49371 13.2 13:45aa48bb:::benchmark_data_smithi001_306460_object49370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.597451+0000", 2024-09-14T04:32:46.463 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.81722644500002, 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4761639679999998, 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49371 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597451+0000", 2024-09-14T04:32:46.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597451+0000", 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597450+0000", 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597524+0000", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4474999999999994e-05 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597535+0000", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.012e-05 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597541+0000", 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8050000000000001e-06 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597591+0000", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9169999999999998e-05 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.597599+0000", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8720000000000008e-06 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551429+0000", 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95382939499999997 2024-09-14T04:32:46.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551436+0000", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3200000000000002e-06 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063741+0000", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5123044750000001 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063830+0000", 2024-09-14T04:32:46.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8877999999999998e-05 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.063919+0000", 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9232000000000002e-05 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.068810+0000", 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048915479999999999 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.071383+0000", 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002572581 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.071494+0000", 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111121 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.071547+0000", 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2543999999999997e-05 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.073615+0000", 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020683049999999999 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49387 13.2 13:4db1c542:::benchmark_data_smithi001_306460_object49386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.609647+0000", 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.80503025600001, 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4659605770000002, 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.471 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49387 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609647+0000", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609647+0000", 2024-09-14T04:32:46.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609645+0000", 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609694+0000", 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9018999999999999e-05 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609711+0000", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6818999999999999e-05 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609718+0000", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1720000000000001e-06 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609773+0000", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4985000000000003e-05 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.609783+0000", 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0880000000000001e-06 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551458+0000", 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94167531599999998 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551464+0000", 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5509999999999996e-06 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.065298+0000", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.513833896 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.065434+0000", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013606999999999999 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.065562+0000", 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127649 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.071394+0000", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058316599999999998 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.071447+0000", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2864000000000001e-05 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.073671+0000", 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022247529999999999 2024-09-14T04:32:46.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.073731+0000", 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0056999999999998e-05 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.075608+0000", 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018762920000000001 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.478 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49393 13.2 13:48ea07fc:::benchmark_data_smithi001_306460_object49392:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.479 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.615259+0000", 2024-09-14T04:32:46.479 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.79941873500002, 2024-09-14T04:32:46.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4789182479999998, 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49393 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615259+0000", 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615259+0000", 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615258+0000", 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615304+0000", 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6213e-05 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615314+0000", 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0377000000000001e-05 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615322+0000", 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4510000000000003e-06 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615378+0000", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6526000000000001e-05 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.615385+0000", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6900000000000003e-06 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551486+0000", 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93610121199999996 2024-09-14T04:32:46.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551492+0000", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2609999999999999e-06 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.075648+0000", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5241556090000001 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.075794+0000", 2024-09-14T04:32:46.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000145931 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.075936+0000", 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014194500000000001 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.081277+0000", 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053414120000000002 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.091892+0000", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010615021000000001 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092073+0000", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000181046 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092108+0000", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4932999999999999e-05 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.094177+0000", 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002068566 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49417 13.2 13:4d87192d:::benchmark_data_smithi001_306460_object49416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.630657+0000", 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.784020656, 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4656376249999998, 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49417 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630657+0000", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630657+0000", 2024-09-14T04:32:46.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630655+0000", 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630712+0000", 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7246e-05 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630729+0000", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6733999999999999e-05 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630737+0000", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4059999999999999e-06 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630800+0000", 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3533000000000003e-05 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.630811+0000", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1151e-05 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551515+0000", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.92070353699999996 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551522+0000", 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7519999999999996e-06 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.077685+0000", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5261626260000001 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.077807+0000", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012176 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.077947+0000", 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139911 2024-09-14T04:32:46.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.086896+0000", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089490709999999994 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.091994+0000", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050982550000000003 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.094284+0000", 2024-09-14T04:32:46.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022898620000000001 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.094343+0000", 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9157999999999998e-05 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.096294+0000", 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001951176 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49429 13.2 13:4f5c8dc5:::benchmark_data_smithi001_306460_object49428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.637914+0000", 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.776763109, 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4698164029999998, 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49429 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637914+0000", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637914+0000", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637913+0000", 2024-09-14T04:32:46.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637973+0000", 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0259e-05 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637989+0000", 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6050999999999999e-05 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.637996+0000", 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9450000000000004e-06 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.638061+0000", 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4553000000000003e-05 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.638071+0000", 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0662999999999999e-05 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551544+0000", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91347241599999995 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551550+0000", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2419999999999997e-06 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.079464+0000", 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5279138909999999 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.079587+0000", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012295700000000001 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.079723+0000", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013653600000000001 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.086936+0000", 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072122430000000001 2024-09-14T04:32:46.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092002+0000", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050662579999999997 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.105707+0000", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013704638999999999 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.105769+0000", 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2465999999999995e-05 2024-09-14T04:32:46.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.107731+0000", 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001961656 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49444 13.2 13:4db29bb4:::benchmark_data_smithi001_306460_object49443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.648955+0000", 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.765722149, 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4602031970000002, 2024-09-14T04:32:46.502 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49444 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.648955+0000", 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.648955+0000", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.648954+0000", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.649003+0000", 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8399999999999997e-05 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.649014+0000", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1297e-05 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.649021+0000", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.357e-06 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.649084+0000", 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2500000000000001e-05 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.649093+0000", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7049999999999994e-06 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551569+0000", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90247549100000002 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551575+0000", 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2519999999999996e-06 2024-09-14T04:32:46.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.081303+0000", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5297273709999999 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.081443+0000", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014000699999999999 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.081604+0000", 2024-09-14T04:32:46.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000161435 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.091935+0000", 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010330545 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092010+0000", 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5791999999999995e-05 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.107817+0000", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015806673 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.107853+0000", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6431999999999999e-05 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.109158+0000", 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013048910000000001 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49447 13.2 13:46c569bb:::benchmark_data_smithi001_306460_object49446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.651088+0000", 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.763589825, 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.460434394, 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49447 2024-09-14T04:32:46.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651088+0000", 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651088+0000", 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651087+0000", 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651166+0000", 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9624999999999997e-05 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651178+0000", 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1155000000000001e-05 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651185+0000", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4050000000000003e-06 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651231+0000", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6436000000000001e-05 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.651240+0000", 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8880000000000003e-06 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551597+0000", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90035717199999998 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551604+0000", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4359999999999999e-06 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.083173+0000", 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.531568936 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.083317+0000", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014375799999999999 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.083495+0000", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017874399999999999 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.091985+0000", 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084894730000000009 2024-09-14T04:32:46.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092018+0000", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2789000000000001e-05 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.109260+0000", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017242652000000001 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.109585+0000", 2024-09-14T04:32:46.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032463099999999998 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.111522+0000", 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019370679999999999 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49471 13.2 13:4eeab0c1:::benchmark_data_smithi001_306460_object49470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.676840+0000", 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.73783748099999, 2024-09-14T04:32:46.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.436689383, 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49471 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676840+0000", 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676840+0000", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676834+0000", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999938 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676882+0000", 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8469e-05 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676897+0000", 2024-09-14T04:32:46.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4695000000000001e-05 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676905+0000", 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3070000000000003e-06 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676964+0000", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9296999999999998e-05 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.676977+0000", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2876e-05 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551627+0000", 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.87464951599999996 2024-09-14T04:32:46.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551634+0000", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2339999999999997e-06 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.085035+0000", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.533401257 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.085150+0000", 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114411 2024-09-14T04:32:46.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.085298+0000", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014862999999999999 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092025+0000", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067264830000000001 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.094237+0000", 2024-09-14T04:32:46.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022122209999999999 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.111589+0000", 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017352067999999998 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.111645+0000", 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5729000000000001e-05 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.113529+0000", 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018844109999999999 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49476 13.2 13:44b00732:::benchmark_data_smithi001_306460_object49475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.681024+0000", 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.73365375700001, 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4345280059999999, 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.526 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49476 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681024+0000", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681024+0000", 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681022+0000", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681099+0000", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6507000000000005e-05 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681111+0000", 2024-09-14T04:32:46.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2524000000000001e-05 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681119+0000", 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6190000000000002e-06 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681171+0000", 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1826000000000002e-05 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.681178+0000", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9800000000000001e-06 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551670+0000", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.87049273100000002 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551686+0000", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5218000000000001e-05 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.086959+0000", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5352731419999999 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.087092+0000", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013325500000000001 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.087256+0000", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016411099999999999 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092033+0000", 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004777122 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.096368+0000", 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043344990000000003 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.113602+0000", 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017234638 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.113656+0000", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3086e-05 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.115552+0000", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018960279999999999 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49484 13.2 13:4dbac7c1:::benchmark_data_smithi001_306460_object49483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.692735+0000", 2024-09-14T04:32:46.533 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.72194237799999, 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4296805899999998, 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49484 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692735+0000", 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692735+0000", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692733+0000", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692773+0000", 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9557000000000001e-05 2024-09-14T04:32:46.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692787+0000", 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4042e-05 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692795+0000", 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7200000000000006e-06 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692851+0000", 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5974999999999999e-05 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.692863+0000", 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2053e-05 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551728+0000", 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85886565199999998 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551737+0000", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3610000000000001e-06 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.088860+0000", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.537123566 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.089003+0000", 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000142946 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.089171+0000", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016795100000000001 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.092042+0000", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028711079999999998 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.096434+0000", 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043916340000000002 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.120307+0000", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023872951 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.120368+0000", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1434999999999997e-05 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.122416+0000", 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002047199 2024-09-14T04:32:46.540 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:49490 13.2 13:4a58a86e:::benchmark_data_smithi001_306460_object49489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e413)", 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:25:34.704553+0000", 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 431.71012454599997, 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4195559859999998, 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.541 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 49490 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704553+0000", 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704553+0000", 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704551+0000", 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704592+0000", 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1121999999999997e-05 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704609+0000", 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6368999999999999e-05 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704616+0000", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3409999999999997e-06 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704669+0000", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2787999999999997e-05 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:34.704684+0000", 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4997999999999999e-05 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551763+0000", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84707937499999997 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:35.551771+0000", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2749999999999995e-06 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.090417+0000", 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.538646027 2024-09-14T04:32:46.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.090524+0000", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106989 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.090737+0000", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021280800000000001 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.094248+0000", 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00351039 2024-09-14T04:32:46.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.096502+0000", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022547449999999998 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.122488+0000", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025985194999999999 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.122552+0000", 2024-09-14T04:32:46.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4689000000000004e-05 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:25:37.124109+0000", 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015565360000000001 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229207 13.17 13:ea6b5d0a:::benchmark_data_smithi001_306460_object110963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:44.211681+0000", 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 242.20299640600001, 2024-09-14T04:32:46.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84863769899999997, 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229207 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211681+0000", 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211681+0000", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211680+0000", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211686+0000", 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7039999999999997e-06 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211701+0000", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4867e-05 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.211709+0000", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2819999999999996e-06 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.212008+0000", 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029889100000000002 2024-09-14T04:32:46.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.212031+0000", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2616000000000002e-05 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.055234+0000", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84320311999999997 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.055371+0000", 2024-09-14T04:32:46.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013727100000000001 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.055664+0000", 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029267300000000002 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.058661+0000", 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029972509999999998 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.060038+0000", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013772260000000001 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.060224+0000", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018550199999999999 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.060279+0000", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4939999999999999e-05 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.060319+0000", 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9768999999999998e-05 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229224 13.17 13:ebf4311f:::benchmark_data_smithi001_306460_object110980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:44.225585+0000", 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 242.189092776, 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.83862775000000001, 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.555 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229224 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225585+0000", 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225585+0000", 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225583+0000", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225589+0000", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9309999999999996e-06 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225600+0000", 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0462000000000001e-05 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225606+0000", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.489e-06 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225655+0000", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9033999999999998e-05 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.225664+0000", 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5220000000000007e-06 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.057354+0000", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.83169017099999998 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.057472+0000", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011857399999999999 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.057714+0000", 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024170300000000001 2024-09-14T04:32:46.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.060196+0000", 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024818489999999999 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062005+0000", 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001809094 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064139+0000", 2024-09-14T04:32:46.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002134052 2024-09-14T04:32:46.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064185+0000", 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6378000000000002e-05 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064212+0000", 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6811000000000001e-05 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.562 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229249 13.17 13:edee6f9d:::benchmark_data_smithi001_306460_object111005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:44.245538+0000", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 242.169139126, 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.81879225, 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229249 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245538+0000", 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245538+0000", 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245537+0000", 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245543+0000", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5629999999999997e-06 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245556+0000", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2421e-05 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245563+0000", 2024-09-14T04:32:46.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0350000000000004e-06 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245621+0000", 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8802999999999999e-05 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.245630+0000", 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8019999999999998e-06 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.058698+0000", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.81306770900000003 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.058777+0000", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8566000000000006e-05 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.058999+0000", 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000222065 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062052+0000", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030538549999999999 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062090+0000", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7852000000000003e-05 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064227+0000", 2024-09-14T04:32:46.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021365160000000002 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064281+0000", 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4245e-05 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064330+0000", 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9382000000000003e-05 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.46076.0:229284 13.17 13:ef536507:::benchmark_data_smithi001_306460_object111040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2024-09-14T04:28:44.275787+0000", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 242.138890683, 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.79365667200000001, 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.46076", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 229284 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2024-09-14T04:32:46.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275787+0000", 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275787+0000", 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275785+0000", 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275792+0000", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2820000000000002e-06 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275807+0000", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5671e-05 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275815+0000", 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.233e-06 2024-09-14T04:32:46.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275881+0000", 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6492e-05 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:44.275892+0000", 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0553999999999999e-05 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2024-09-14T04:32:46.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062161+0000", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.78626930399999995 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062266+0000", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104749 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.062545+0000", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027959799999999999 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.064490+0000", 2024-09-14T04:32:46.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019452289999999999 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.066547+0000", 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020562340000000001 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.069322+0000", 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027750180000000002 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2024-09-14T04:32:46.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.069397+0000", 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5160000000000005e-05 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2024-09-14T04:28:45.069443+0000", 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6449999999999997e-05 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-14T04:32:46.576 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-14T04:32:46.597 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:46.699 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:46.731 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:46.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:46.871 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:46.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:46] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:32:46.972 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.001 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:47.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:47.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:47.147 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:47.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:47.284 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:47.386 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:47.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:47.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:47.549 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:47.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.691 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:47.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:47.792 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:47.820 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:47.922 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:47.951 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:47.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:48.053 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:48.084 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:48.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:48.224 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:48.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:48.363 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:48.465 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:48.493 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:48.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:48.644 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:48.653 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:32:48.747 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:48.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.783 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:48.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:48.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:48.916 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:48.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:48.945 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Removed 98943 objects 2024-09-14T04:32:48.945 INFO:tasks.radosbench.radosbench.0.smithi001.stdout:Clean up completed and total clean up time :106.488 2024-09-14T04:32:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:48 smithi001 ceph-mon[112165]: pgmap v1714: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 232 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 4621/11175 objects degraded (41.351%); 2072/11175 objects misplaced (18.541%) 2024-09-14T04:32:48.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:48 smithi001 ceph-mon[107399]: pgmap v1714: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 232 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 4621/11175 objects degraded (41.351%); 2072/11175 objects misplaced (18.541%) 2024-09-14T04:32:49.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:49.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:49.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:48 smithi012 ceph-mon[101180]: pgmap v1714: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 232 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 4621/11175 objects degraded (41.351%); 2072/11175 objects misplaced (18.541%) 2024-09-14T04:32:49.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:49.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:49.266 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:49.290 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:49.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:49.392 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.415 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:49.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:49.516 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:49.539 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:49.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:49.641 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:49.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:49.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:49.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:49.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:49.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:49 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/11175 objects degraded (41.351%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:49 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/11175 objects degraded (41.351%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:50.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:50.038 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:50.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:49 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/11175 objects degraded (41.351%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:50.140 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:50.164 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:50.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:50.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:50.341 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:32:50.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:50.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:50.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:50.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:50.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:50.663 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:50.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:50.785 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:50.794 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:50.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:50.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:50.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:50.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:50 smithi001 ceph-mon[112165]: pgmap v1715: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 212 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 4621/10215 objects degraded (45.237%); 1900/10215 objects misplaced (18.600%); 0 B/s, 38 objects/s recovering 2024-09-14T04:32:50.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:50 smithi001 ceph-mon[107399]: pgmap v1715: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 212 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 4621/10215 objects degraded (45.237%); 1900/10215 objects misplaced (18.600%); 0 B/s, 38 objects/s recovering 2024-09-14T04:32:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:51.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:51.042 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:50 smithi012 ceph-mon[101180]: pgmap v1715: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 212 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 4621/10215 objects degraded (45.237%); 1900/10215 objects misplaced (18.600%); 0 B/s, 38 objects/s recovering 2024-09-14T04:32:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:51.144 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:51.170 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:51.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:51.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:51.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:51.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:51.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:51.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:51.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:51.770 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:51.796 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:51.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:51.798 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:32:51.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:51.921 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:51.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:52.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:52.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:51 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:51 smithi012 ceph-mon[101180]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 37 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 4621/1839 objects degraded (251.278%); 1900/1839 objects misplaced (103.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:32:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:51 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/161381394' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:51 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:52.170 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:52.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[107399]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 37 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 4621/1839 objects degraded (251.278%); 1900/1839 objects misplaced (103.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/161381394' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[112165]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 37 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 4621/1839 objects degraded (251.278%); 1900/1839 objects misplaced (103.317%); 0 B/s, 27 objects/s recovering 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/161381394' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:51 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:32:52.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:52.295 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:52.397 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:52.421 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:52.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:52.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:52.546 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:52.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:52.670 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:52.737 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2024-09-14T04:32:52.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2024-09-14T04:32:52.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:52.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.795 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:52.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:52.919 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:52.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:52.961 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:32:52.961 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:52 smithi001 ceph-mon[107399]: osdmap e538: 8 total, 8 up, 6 in 2024-09-14T04:32:52.961 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:32:52.962 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:52 smithi001 ceph-mon[112165]: osdmap e538: 8 total, 8 up, 6 in 2024-09-14T04:32:53.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:53.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:53.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:32:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:52 smithi012 ceph-mon[101180]: osdmap e538: 8 total, 8 up, 6 in 2024-09-14T04:32:53.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:53.167 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:53.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:53.293 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:53.395 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:53.418 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:53.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:53.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:53.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:53.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:53.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:53.790 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:53.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:53.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[112165]: osdmap e539: 8 total, 8 up, 6 in 2024-09-14T04:32:53.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:53.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[112165]: pgmap v1719: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 649 op/s; 4621/1128 objects degraded (409.663%); 1900/1128 objects misplaced (168.440%); 0 B/s, 21 objects/s recovering 2024-09-14T04:32:53.848 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[112165]: osdmap e540: 8 total, 8 up, 6 in 2024-09-14T04:32:53.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:53.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[107399]: osdmap e539: 8 total, 8 up, 6 in 2024-09-14T04:32:53.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:53.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[107399]: pgmap v1719: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 649 op/s; 4621/1128 objects degraded (409.663%); 1900/1128 objects misplaced (168.440%); 0 B/s, 21 objects/s recovering 2024-09-14T04:32:53.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:53 smithi001 ceph-mon[107399]: osdmap e540: 8 total, 8 up, 6 in 2024-09-14T04:32:53.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:53.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:53.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:53.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:54.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:54.041 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:53 smithi012 ceph-mon[101180]: osdmap e539: 8 total, 8 up, 6 in 2024-09-14T04:32:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:32:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:53 smithi012 ceph-mon[101180]: pgmap v1719: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 12 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 649 op/s; 4621/1128 objects degraded (409.663%); 1900/1128 objects misplaced (168.440%); 0 B/s, 21 objects/s recovering 2024-09-14T04:32:54.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:53 smithi012 ceph-mon[101180]: osdmap e540: 8 total, 8 up, 6 in 2024-09-14T04:32:54.143 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:54.167 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:54.269 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:54.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:54.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:54.370 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:32:54.370 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [], 2024-09-14T04:32:54.370 INFO:teuthology.orchestra.run.smithi001.stdout: "num_ops": 0 2024-09-14T04:32:54.370 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:32:54.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:54.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:54.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:54.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:54.543 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:54.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:54.668 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:54.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:54.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:54.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:54.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:54.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:54.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:54.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: 14.9 scrub starts 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: 14.9 scrub ok 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 537'12508 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 537'12468 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 4621/1128 objects degraded (409.663%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[107399]: osdmap e541: 8 total, 8 up, 6 in 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: 14.9 scrub starts 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: 14.9 scrub ok 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 537'12508 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 537'12468 2024-09-14T04:32:54.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 4621/1128 objects degraded (409.663%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:54.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:54 smithi001 ceph-mon[112165]: osdmap e541: 8 total, 8 up, 6 in 2024-09-14T04:32:55.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:55.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:55.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: 14.9 scrub starts 2024-09-14T04:32:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: 14.9 scrub ok 2024-09-14T04:32:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 537'12508 2024-09-14T04:32:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 537'12468 2024-09-14T04:32:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 4621/1128 objects degraded (409.663%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:54 smithi012 ceph-mon[101180]: osdmap e541: 8 total, 8 up, 6 in 2024-09-14T04:32:55.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:55.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:55.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:55.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:55.308 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is set 2024-09-14T04:32:55.337 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:32:55.338 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:32:55.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:55.415 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:55.517 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:55.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:55.588 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:55.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.667 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:55.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:55.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:55.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:55.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:55.915 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:55.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:55.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[107399]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[107399]: osdmap e542: 8 total, 8 up, 6 in 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[107399]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 3 peering, 10 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 480/1086 objects degraded (44.199%); 882/1086 objects misplaced (81.215%); 0 B/s, 464 objects/s recovering 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[112165]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[112165]: osdmap e542: 8 total, 8 up, 6 in 2024-09-14T04:32:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:55 smithi001 ceph-mon[112165]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 3 peering, 10 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 480/1086 objects degraded (44.199%); 882/1086 objects misplaced (81.215%); 0 B/s, 464 objects/s recovering 2024-09-14T04:32:56.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:56.039 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:56.062 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:56.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:32:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:55 smithi012 ceph-mon[101180]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:32:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/470475896' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:32:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:55 smithi012 ceph-mon[101180]: osdmap e542: 8 total, 8 up, 6 in 2024-09-14T04:32:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:55 smithi012 ceph-mon[101180]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 3 peering, 10 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 22 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 480/1086 objects degraded (44.199%); 882/1086 objects misplaced (81.215%); 0 B/s, 464 objects/s recovering 2024-09-14T04:32:56.141 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:56.166 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:56.267 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:56.292 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:56.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:56.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:56.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:56.540 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:56.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:56.641 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:56.665 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:56.767 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:56.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:56.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:56.914 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:56.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:56.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:32:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:32:56] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-14T04:32:57.016 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:32:57.039 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:32:57.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:57.163 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:57.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:57.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:57.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:57.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.414 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:57.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:57.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:57.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:57.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:57 smithi012 ceph-mon[101180]: 14.10 scrub starts 2024-09-14T04:32:57.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:57 smithi012 ceph-mon[101180]: osdmap e543: 8 total, 8 up, 6 in 2024-09-14T04:32:57.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:57 smithi001 ceph-mon[107399]: 14.10 scrub starts 2024-09-14T04:32:57.607 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:57 smithi001 ceph-mon[107399]: osdmap e543: 8 total, 8 up, 6 in 2024-09-14T04:32:57.607 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:57 smithi001 ceph-mon[112165]: 14.10 scrub starts 2024-09-14T04:32:57.607 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:57 smithi001 ceph-mon[112165]: osdmap e543: 8 total, 8 up, 6 in 2024-09-14T04:32:57.613 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:32:57.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.664 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:57.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:57.765 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:57.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:57.892 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:57.914 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:57.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:58.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:58.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:32:58.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:58.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:58.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:58.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:58.390 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:58.414 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:58 smithi001 ceph-mon[107399]: osdmap e544: 8 total, 8 up, 6 in 2024-09-14T04:32:58.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:58 smithi001 ceph-mon[107399]: pgmap v1726: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 201 objects/s recovering 2024-09-14T04:32:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:58 smithi001 ceph-mon[112165]: osdmap e544: 8 total, 8 up, 6 in 2024-09-14T04:32:58.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:58 smithi001 ceph-mon[112165]: pgmap v1726: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 201 objects/s recovering 2024-09-14T04:32:58.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:58.538 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:58 smithi012 ceph-mon[101180]: osdmap e544: 8 total, 8 up, 6 in 2024-09-14T04:32:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:58 smithi012 ceph-mon[101180]: pgmap v1726: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 201 objects/s recovering 2024-09-14T04:32:58.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:58.662 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:58.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:32:58.787 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:32:58.889 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:58.912 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:58.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:32:59.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:32:59.037 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:32:59.139 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:32:59.162 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:59.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:32:59.262 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:32:59.264 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:32:59.286 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:32:59.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:32:59.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:32:59.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:59.534 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:59.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:59 smithi012 ceph-mon[101180]: osdmap e545: 8 total, 8 up, 6 in 2024-09-14T04:32:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:32:59 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 472/1047 objects degraded (45.081%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:59.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:59.658 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:59.720 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:59 smithi001 ceph-mon[112165]: osdmap e545: 8 total, 8 up, 6 in 2024-09-14T04:32:59.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:32:59 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 472/1047 objects degraded (45.081%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:59 smithi001 ceph-mon[107399]: osdmap e545: 8 total, 8 up, 6 in 2024-09-14T04:32:59.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:32:59 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 472/1047 objects degraded (45.081%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:32:59.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:32:59.783 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:32:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:32:59.884 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:32:59.908 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:32:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:32:59.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:00.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:00.032 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:00.134 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:00.158 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:00.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:00.286 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[112165]: 14.13 scrub starts 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[112165]: osdmap e546: 8 total, 8 up, 6 in 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[112165]: 14.11 scrub starts 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[112165]: pgmap v1729: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 1 objects/s recovering 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/794026905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[107399]: 14.13 scrub starts 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[107399]: osdmap e546: 8 total, 8 up, 6 in 2024-09-14T04:33:00.363 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[107399]: 14.11 scrub starts 2024-09-14T04:33:00.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[107399]: pgmap v1729: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 1 objects/s recovering 2024-09-14T04:33:00.364 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:00 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/794026905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:00.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:00.410 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:00.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:00.535 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:00 smithi012 ceph-mon[101180]: 14.13 scrub starts 2024-09-14T04:33:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:00 smithi012 ceph-mon[101180]: osdmap e546: 8 total, 8 up, 6 in 2024-09-14T04:33:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:00 smithi012 ceph-mon[101180]: 14.11 scrub starts 2024-09-14T04:33:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:00 smithi012 ceph-mon[101180]: pgmap v1729: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovery_wait+remapped, 2 peering, 9 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 19 KiB/s rd, 8 op/s; 472/1047 objects degraded (45.081%); 882/1047 objects misplaced (84.241%); 0 B/s, 1 objects/s recovering 2024-09-14T04:33:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:00 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/794026905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:00.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:00.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:00.761 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:33:00.764 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:33:00.765 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:33:00.784 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:00.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:00.909 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:00.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:01.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:01.034 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:01.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.159 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:01.260 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:01.283 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:01.384 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:01.406 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:01.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:01 smithi001 ceph-mon[107399]: 14.16 scrub starts 2024-09-14T04:33:01.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:01 smithi001 ceph-mon[112165]: 14.16 scrub starts 2024-09-14T04:33:01.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:01.529 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:01.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:01 smithi012 ceph-mon[101180]: 14.16 scrub starts 2024-09-14T04:33:01.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:01.653 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:01.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:01.777 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:01.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:01.901 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:01.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:02.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:02.027 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:02.057 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:02.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:02.155 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:02.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:02.279 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:02.338 INFO:teuthology.orchestra.run.smithi001.stderr:pool 'unique_pool_9' removed 2024-09-14T04:33:02.380 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:02.403 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:02.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:02.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:02.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:02 smithi012 ceph-mon[101180]: 14.18 scrub starts 2024-09-14T04:33:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:02 smithi012 ceph-mon[101180]: pgmap v1730: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 14 KiB/s rd, 6 op/s; 470/1047 objects degraded (44.890%); 0 B/s, 274 objects/s recovering 2024-09-14T04:33:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:33:02.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:02.652 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:02.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[112165]: 14.18 scrub starts 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[112165]: pgmap v1730: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 14 KiB/s rd, 6 op/s; 470/1047 objects degraded (44.890%); 0 B/s, 274 objects/s recovering 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[107399]: 14.18 scrub starts 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[107399]: pgmap v1730: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 21 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 14 KiB/s rd, 6 op/s; 470/1047 objects degraded (44.890%); 0 B/s, 274 objects/s recovering 2024-09-14T04:33:02.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:33:02.755 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:02.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:02.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:02.902 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:02.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:03.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:03.027 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:03.129 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:03.152 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:03.254 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:03.276 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:03.378 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:03.400 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:03.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[112165]: 14.3 scrub starts 2024-09-14T04:33:03.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:33:03.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[112165]: osdmap e547: 8 total, 8 up, 6 in 2024-09-14T04:33:03.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[112165]: 14.6 scrub starts 2024-09-14T04:33:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[107399]: 14.3 scrub starts 2024-09-14T04:33:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:33:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[107399]: osdmap e547: 8 total, 8 up, 6 in 2024-09-14T04:33:03.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:03 smithi001 ceph-mon[107399]: 14.6 scrub starts 2024-09-14T04:33:03.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:03.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:33:03.526 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:03 smithi012 ceph-mon[101180]: 14.3 scrub starts 2024-09-14T04:33:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2472229281' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:33:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:03 smithi012 ceph-mon[101180]: osdmap e547: 8 total, 8 up, 6 in 2024-09-14T04:33:03.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:03 smithi012 ceph-mon[101180]: 14.6 scrub starts 2024-09-14T04:33:03.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:03.651 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:03.753 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.778 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:03.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:03.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:03.905 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:33:03.906 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:33:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:03.909 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:03.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:04.010 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:04.034 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:04.136 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.159 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:04.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:04.287 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:04.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:04.416 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:04.518 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:04.541 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:04.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:04 smithi012 ceph-mon[101180]: pgmap v1732: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:04 smithi012 ceph-mon[101180]: Health check update: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:33:04.643 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.671 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:04.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:04 smithi001 ceph-mon[107399]: pgmap v1732: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:04.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:04 smithi001 ceph-mon[107399]: Health check update: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:33:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:04 smithi001 ceph-mon[112165]: pgmap v1732: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:04.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:04 smithi001 ceph-mon[112165]: Health check update: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:33:04.773 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:04.775 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1732,"stamp":"2024-09-14T04:33:03.649255+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":404813,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":45,"num_read_kb":100,"num_write":89,"num_write_kb":325,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":2465972,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3188,"ondisk_log_size":3188,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":371,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7292828,"kb_used_data":48896,"kb_used_omap":25,"kb_used_meta":7243814,"kb_avail":555178084,"statfs":{"total":575970213888,"available":568502358016,"internally_reserved":0,"allocated":50069504,"data_stored":28654535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26477,"internal_metadata":7417665683},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-21200896,"num_objects":-324,"num_object_clones":0,"num_object_copies":-972,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-472,"num_objects_misplaced":-882,"num_objects_unfound":0,"num_objects_dirty":-324,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-174632,"num_write_kb":-5335873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-29458,"num_bytes_recovered":-1718025677,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001413"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":898,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338911+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:32:59.338911+0000","last_peered":"2024-09-14T04:32:59.338911+0000","last_clean":"2024-09-14T04:32:59.338911+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:32:59.338911+0000","last_undegraded":"2024-09-14T04:32:59.338911+0000","last_fullsized":"2024-09-14T04:32:59.338911+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325343+0000","last_change":"2024-09-14T04:30:04.281338+0000","last_active":"2024-09-14T04:32:58.325343+0000","last_peered":"2024-09-14T04:32:58.325343+0000","last_clean":"2024-09-14T04:32:58.325343+0000","last_became_active":"2024-09-14T04:30:04.280363+0000","last_became_peered":"2024-09-14T04:30:04.280363+0000","last_unstale":"2024-09-14T04:32:58.325343+0000","last_undegraded":"2024-09-14T04:32:58.325343+0000","last_fullsized":"2024-09-14T04:32:58.325343+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:50:56.470339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":138,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325027+0000","last_change":"2024-09-14T04:29:37.998593+0000","last_active":"2024-09-14T04:32:58.325027+0000","last_peered":"2024-09-14T04:32:58.325027+0000","last_clean":"2024-09-14T04:32:58.325027+0000","last_became_active":"2024-09-14T04:29:37.991857+0000","last_became_peered":"2024-09-14T04:29:37.991857+0000","last_unstale":"2024-09-14T04:32:58.325027+0000","last_undegraded":"2024-09-14T04:32:58.325027+0000","last_fullsized":"2024-09-14T04:32:58.325027+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:44:08.110065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":865,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324005+0000","last_change":"2024-09-14T04:32:55.316666+0000","last_active":"2024-09-14T04:32:58.324005+0000","last_peered":"2024-09-14T04:32:58.324005+0000","last_clean":"2024-09-14T04:32:58.324005+0000","last_became_active":"2024-09-14T04:32:55.316301+0000","last_became_peered":"2024-09-14T04:32:55.316301+0000","last_unstale":"2024-09-14T04:32:58.324005+0000","last_undegraded":"2024-09-14T04:32:58.324005+0000","last_fullsized":"2024-09-14T04:32:58.324005+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:15:16.745472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325423+0000","last_change":"2024-09-14T04:29:18.666490+0000","last_active":"2024-09-14T04:32:58.325423+0000","last_peered":"2024-09-14T04:32:58.325423+0000","last_clean":"2024-09-14T04:32:58.325423+0000","last_became_active":"2024-09-14T04:29:18.666113+0000","last_became_peered":"2024-09-14T04:29:18.666113+0000","last_unstale":"2024-09-14T04:32:58.325423+0000","last_undegraded":"2024-09-14T04:32:58.325423+0000","last_fullsized":"2024-09-14T04:32:58.325423+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:30.604568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":721,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338024+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:32:59.338024+0000","last_peered":"2024-09-14T04:32:59.338024+0000","last_clean":"2024-09-14T04:32:59.338024+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:32:59.338024+0000","last_undegraded":"2024-09-14T04:32:59.338024+0000","last_fullsized":"2024-09-14T04:32:59.338024+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":997,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.330567+0000","last_change":"2024-09-14T04:30:07.304986+0000","last_active":"2024-09-14T04:32:59.330567+0000","last_peered":"2024-09-14T04:32:59.330567+0000","last_clean":"2024-09-14T04:32:59.330567+0000","last_became_active":"2024-09-14T04:30:07.304452+0000","last_became_peered":"2024-09-14T04:30:07.304452+0000","last_unstale":"2024-09-14T04:32:59.330567+0000","last_undegraded":"2024-09-14T04:32:59.330567+0000","last_fullsized":"2024-09-14T04:32:59.330567+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:59.521970+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":707,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339884+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:32:59.339884+0000","last_peered":"2024-09-14T04:32:59.339884+0000","last_clean":"2024-09-14T04:32:59.339884+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:32:59.339884+0000","last_undegraded":"2024-09-14T04:32:59.339884+0000","last_fullsized":"2024-09-14T04:32:59.339884+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":126,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.323990+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:32:58.323990+0000","last_peered":"2024-09-14T04:32:58.323990+0000","last_clean":"2024-09-14T04:32:58.323990+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:32:58.323990+0000","last_undegraded":"2024-09-14T04:32:58.323990+0000","last_fullsized":"2024-09-14T04:32:58.323990+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":704,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328731+0000","last_change":"2024-09-14T04:30:07.297746+0000","last_active":"2024-09-14T04:32:58.328731+0000","last_peered":"2024-09-14T04:32:58.328731+0000","last_clean":"2024-09-14T04:32:58.328731+0000","last_became_active":"2024-09-14T04:30:07.297186+0000","last_became_peered":"2024-09-14T04:30:07.297186+0000","last_unstale":"2024-09-14T04:32:58.328731+0000","last_undegraded":"2024-09-14T04:32:58.328731+0000","last_fullsized":"2024-09-14T04:32:58.328731+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:59:40.613090+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":293,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.331033+0000","last_change":"2024-09-14T04:30:04.286496+0000","last_active":"2024-09-14T04:32:59.331033+0000","last_peered":"2024-09-14T04:32:59.331033+0000","last_clean":"2024-09-14T04:32:59.331033+0000","last_became_active":"2024-09-14T04:30:04.282753+0000","last_became_peered":"2024-09-14T04:30:04.282753+0000","last_unstale":"2024-09-14T04:32:59.331033+0000","last_undegraded":"2024-09-14T04:32:59.331033+0000","last_fullsized":"2024-09-14T04:32:59.331033+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:39.799809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":108,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.327860+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:32:59.327860+0000","last_peered":"2024-09-14T04:32:59.327860+0000","last_clean":"2024-09-14T04:32:59.327860+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:32:59.327860+0000","last_undegraded":"2024-09-14T04:32:59.327860+0000","last_fullsized":"2024-09-14T04:32:59.327860+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":833,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325650+0000","last_change":"2024-09-14T04:30:07.303908+0000","last_active":"2024-09-14T04:32:58.325650+0000","last_peered":"2024-09-14T04:32:58.325650+0000","last_clean":"2024-09-14T04:32:58.325650+0000","last_became_active":"2024-09-14T04:30:07.302591+0000","last_became_peered":"2024-09-14T04:30:07.302591+0000","last_unstale":"2024-09-14T04:32:58.325650+0000","last_undegraded":"2024-09-14T04:32:58.325650+0000","last_fullsized":"2024-09-14T04:32:58.325650+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:51.333058+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":121,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325184+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:32:58.325184+0000","last_peered":"2024-09-14T04:32:58.325184+0000","last_clean":"2024-09-14T04:32:58.325184+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:32:58.325184+0000","last_undegraded":"2024-09-14T04:32:58.325184+0000","last_fullsized":"2024-09-14T04:32:58.325184+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.323804+0000","last_change":"2024-09-14T04:32:52.814891+0000","last_active":"2024-09-14T04:32:58.323804+0000","last_peered":"2024-09-14T04:32:58.323804+0000","last_clean":"2024-09-14T04:32:58.323804+0000","last_became_active":"2024-09-14T04:32:52.814547+0000","last_became_peered":"2024-09-14T04:32:52.814547+0000","last_unstale":"2024-09-14T04:32:58.323804+0000","last_undegraded":"2024-09-14T04:32:58.323804+0000","last_fullsized":"2024-09-14T04:32:58.323804+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:43:14.342218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1255,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.331154+0000","last_change":"2024-09-14T04:28:22.698300+0000","last_active":"2024-09-14T04:32:59.331154+0000","last_peered":"2024-09-14T04:32:59.331154+0000","last_clean":"2024-09-14T04:32:59.331154+0000","last_became_active":"2024-09-14T04:28:22.697894+0000","last_became_peered":"2024-09-14T04:28:22.697894+0000","last_unstale":"2024-09-14T04:32:59.331154+0000","last_undegraded":"2024-09-14T04:32:59.331154+0000","last_fullsized":"2024-09-14T04:32:59.331154+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:19.688955+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":134,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324776+0000","last_change":"2024-09-14T04:30:04.302042+0000","last_active":"2024-09-14T04:32:58.324776+0000","last_peered":"2024-09-14T04:32:58.324776+0000","last_clean":"2024-09-14T04:32:58.324776+0000","last_became_active":"2024-09-14T04:30:04.281337+0000","last_became_peered":"2024-09-14T04:30:04.281337+0000","last_unstale":"2024-09-14T04:32:58.324776+0000","last_undegraded":"2024-09-14T04:32:58.324776+0000","last_fullsized":"2024-09-14T04:32:58.324776+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:39.377188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":593,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325144+0000","last_change":"2024-09-14T04:32:52.818598+0000","last_active":"2024-09-14T04:32:58.325144+0000","last_peered":"2024-09-14T04:32:58.325144+0000","last_clean":"2024-09-14T04:32:58.325144+0000","last_became_active":"2024-09-14T04:32:52.818281+0000","last_became_peered":"2024-09-14T04:32:52.818281+0000","last_unstale":"2024-09-14T04:32:58.325144+0000","last_undegraded":"2024-09-14T04:32:58.325144+0000","last_fullsized":"2024-09-14T04:32:58.325144+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:39.775021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":497,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329569+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:32:58.329569+0000","last_peered":"2024-09-14T04:32:58.329569+0000","last_clean":"2024-09-14T04:32:58.329569+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:32:58.329569+0000","last_undegraded":"2024-09-14T04:32:58.329569+0000","last_fullsized":"2024-09-14T04:32:58.329569+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":134,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325743+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:32:58.325743+0000","last_peered":"2024-09-14T04:32:58.325743+0000","last_clean":"2024-09-14T04:32:58.325743+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:32:58.325743+0000","last_undegraded":"2024-09-14T04:32:58.325743+0000","last_fullsized":"2024-09-14T04:32:58.325743+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328712+0000","last_change":"2024-09-14T04:32:52.810817+0000","last_active":"2024-09-14T04:32:58.328712+0000","last_peered":"2024-09-14T04:32:58.328712+0000","last_clean":"2024-09-14T04:32:58.328712+0000","last_became_active":"2024-09-14T04:32:52.809483+0000","last_became_peered":"2024-09-14T04:32:52.809483+0000","last_unstale":"2024-09-14T04:32:58.328712+0000","last_undegraded":"2024-09-14T04:32:58.328712+0000","last_fullsized":"2024-09-14T04:32:58.328712+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:00.995883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":554,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324563+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:32:58.324563+0000","last_peered":"2024-09-14T04:32:58.324563+0000","last_clean":"2024-09-14T04:32:58.324563+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:32:58.324563+0000","last_undegraded":"2024-09-14T04:32:58.324563+0000","last_fullsized":"2024-09-14T04:32:58.324563+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":539,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.330851+0000","last_change":"2024-09-14T04:29:18.673642+0000","last_active":"2024-09-14T04:32:59.330851+0000","last_peered":"2024-09-14T04:32:59.330851+0000","last_clean":"2024-09-14T04:32:59.330851+0000","last_became_active":"2024-09-14T04:29:18.673365+0000","last_became_peered":"2024-09-14T04:29:18.673365+0000","last_unstale":"2024-09-14T04:32:59.330851+0000","last_undegraded":"2024-09-14T04:32:59.330851+0000","last_fullsized":"2024-09-14T04:32:59.330851+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:32.707692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":700,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.343731+0000","last_change":"2024-09-14T04:30:04.292857+0000","last_active":"2024-09-14T04:32:59.343731+0000","last_peered":"2024-09-14T04:32:59.343731+0000","last_clean":"2024-09-14T04:32:59.343731+0000","last_became_active":"2024-09-14T04:30:04.292434+0000","last_became_peered":"2024-09-14T04:30:04.292434+0000","last_unstale":"2024-09-14T04:32:59.343731+0000","last_undegraded":"2024-09-14T04:32:59.343731+0000","last_fullsized":"2024-09-14T04:32:59.343731+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:28.993975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":900,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324158+0000","last_change":"2024-09-14T04:29:40.287295+0000","last_active":"2024-09-14T04:32:58.324158+0000","last_peered":"2024-09-14T04:32:58.324158+0000","last_clean":"2024-09-14T04:32:58.324158+0000","last_became_active":"2024-09-14T04:29:40.286891+0000","last_became_peered":"2024-09-14T04:29:40.286891+0000","last_unstale":"2024-09-14T04:32:58.324158+0000","last_undegraded":"2024-09-14T04:32:58.324158+0000","last_fullsized":"2024-09-14T04:32:58.324158+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:59.160582+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":697,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.343361+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:32:59.343361+0000","last_peered":"2024-09-14T04:32:59.343361+0000","last_clean":"2024-09-14T04:32:59.343361+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:32:59.343361+0000","last_undegraded":"2024-09-14T04:32:59.343361+0000","last_fullsized":"2024-09-14T04:32:59.343361+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":586,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325256+0000","last_change":"2024-09-14T04:32:52.819885+0000","last_active":"2024-09-14T04:32:58.325256+0000","last_peered":"2024-09-14T04:32:58.325256+0000","last_clean":"2024-09-14T04:32:58.325256+0000","last_became_active":"2024-09-14T04:32:52.819034+0000","last_became_peered":"2024-09-14T04:32:52.819034+0000","last_unstale":"2024-09-14T04:32:58.325256+0000","last_undegraded":"2024-09-14T04:32:58.325256+0000","last_fullsized":"2024-09-14T04:32:58.325256+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:52.327969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":375,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.332066+0000","last_change":"2024-09-14T04:32:59.331716+0000","last_active":"2024-09-14T04:32:59.332066+0000","last_peered":"2024-09-14T04:32:59.332066+0000","last_clean":"2024-09-14T04:32:59.332066+0000","last_became_active":"2024-09-14T04:32:59.331294+0000","last_became_peered":"2024-09-14T04:32:59.331294+0000","last_unstale":"2024-09-14T04:32:59.332066+0000","last_undegraded":"2024-09-14T04:32:59.332066+0000","last_fullsized":"2024-09-14T04:32:59.332066+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328877+0000","last_change":"2024-09-14T04:30:04.285245+0000","last_active":"2024-09-14T04:32:58.328877+0000","last_peered":"2024-09-14T04:32:58.328877+0000","last_clean":"2024-09-14T04:32:58.328877+0000","last_became_active":"2024-09-14T04:30:04.284576+0000","last_became_peered":"2024-09-14T04:30:04.284576+0000","last_unstale":"2024-09-14T04:32:58.328877+0000","last_undegraded":"2024-09-14T04:32:58.328877+0000","last_fullsized":"2024-09-14T04:32:58.328877+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:15.875331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":176,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325239+0000","last_change":"2024-09-14T04:30:04.295569+0000","last_active":"2024-09-14T04:32:58.325239+0000","last_peered":"2024-09-14T04:32:58.325239+0000","last_clean":"2024-09-14T04:32:58.325239+0000","last_became_active":"2024-09-14T04:30:04.294571+0000","last_became_peered":"2024-09-14T04:30:04.294571+0000","last_unstale":"2024-09-14T04:32:58.325239+0000","last_undegraded":"2024-09-14T04:32:58.325239+0000","last_fullsized":"2024-09-14T04:32:58.325239+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:40.129021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":943,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.342968+0000","last_change":"2024-09-14T04:32:55.317365+0000","last_active":"2024-09-14T04:32:59.342968+0000","last_peered":"2024-09-14T04:32:59.342968+0000","last_clean":"2024-09-14T04:32:59.342968+0000","last_became_active":"2024-09-14T04:32:55.316994+0000","last_became_peered":"2024-09-14T04:32:55.316994+0000","last_unstale":"2024-09-14T04:32:59.342968+0000","last_undegraded":"2024-09-14T04:32:59.342968+0000","last_fullsized":"2024-09-14T04:32:59.342968+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:17.732488+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324627+0000","last_change":"2024-09-14T04:30:04.291481+0000","last_active":"2024-09-14T04:32:58.324627+0000","last_peered":"2024-09-14T04:32:58.324627+0000","last_clean":"2024-09-14T04:32:58.324627+0000","last_became_active":"2024-09-14T04:30:04.291034+0000","last_became_peered":"2024-09-14T04:30:04.291034+0000","last_unstale":"2024-09-14T04:32:58.324627+0000","last_undegraded":"2024-09-14T04:32:58.324627+0000","last_fullsized":"2024-09-14T04:32:58.324627+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:09.399818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":583,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325362+0000","last_change":"2024-09-14T04:32:52.817581+0000","last_active":"2024-09-14T04:32:58.325362+0000","last_peered":"2024-09-14T04:32:58.325362+0000","last_clean":"2024-09-14T04:32:58.325362+0000","last_became_active":"2024-09-14T04:32:52.817276+0000","last_became_peered":"2024-09-14T04:32:52.817276+0000","last_unstale":"2024-09-14T04:32:58.325362+0000","last_undegraded":"2024-09-14T04:32:58.325362+0000","last_fullsized":"2024-09-14T04:32:58.325362+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:27.119610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":735,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328557+0000","last_change":"2024-09-14T04:29:40.277117+0000","last_active":"2024-09-14T04:32:58.328557+0000","last_peered":"2024-09-14T04:32:58.328557+0000","last_clean":"2024-09-14T04:32:58.328557+0000","last_became_active":"2024-09-14T04:29:40.276702+0000","last_became_peered":"2024-09-14T04:29:40.276702+0000","last_unstale":"2024-09-14T04:32:58.328557+0000","last_undegraded":"2024-09-14T04:32:58.328557+0000","last_fullsized":"2024-09-14T04:32:58.328557+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:19.375908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:19.375908+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:20.933166+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":293,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.329813+0000","last_change":"2024-09-14T04:30:04.298649+0000","last_active":"2024-09-14T04:32:59.329813+0000","last_peered":"2024-09-14T04:32:59.329813+0000","last_clean":"2024-09-14T04:32:59.329813+0000","last_became_active":"2024-09-14T04:30:04.295253+0000","last_became_peered":"2024-09-14T04:30:04.295253+0000","last_unstale":"2024-09-14T04:32:59.329813+0000","last_undegraded":"2024-09-14T04:32:59.329813+0000","last_fullsized":"2024-09-14T04:32:59.329813+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:23.430095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":186,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324368+0000","last_change":"2024-09-14T04:30:04.296998+0000","last_active":"2024-09-14T04:32:58.324368+0000","last_peered":"2024-09-14T04:32:58.324368+0000","last_clean":"2024-09-14T04:32:58.324368+0000","last_became_active":"2024-09-14T04:30:04.296685+0000","last_became_peered":"2024-09-14T04:30:04.296685+0000","last_unstale":"2024-09-14T04:32:58.324368+0000","last_undegraded":"2024-09-14T04:32:58.324368+0000","last_fullsized":"2024-09-14T04:32:58.324368+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:31.564351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":955,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324161+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:32:58.324161+0000","last_peered":"2024-09-14T04:32:58.324161+0000","last_clean":"2024-09-14T04:32:58.324161+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:32:58.324161+0000","last_undegraded":"2024-09-14T04:32:58.324161+0000","last_fullsized":"2024-09-14T04:32:58.324161+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":447,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324911+0000","last_change":"2024-09-14T04:29:38.001028+0000","last_active":"2024-09-14T04:32:58.324911+0000","last_peered":"2024-09-14T04:32:58.324911+0000","last_clean":"2024-09-14T04:32:58.324911+0000","last_became_active":"2024-09-14T04:29:37.987180+0000","last_became_peered":"2024-09-14T04:29:37.987180+0000","last_unstale":"2024-09-14T04:32:58.324911+0000","last_undegraded":"2024-09-14T04:32:58.324911+0000","last_fullsized":"2024-09-14T04:32:58.324911+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:29.081173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":147,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328796+0000","last_change":"2024-09-14T04:29:37.992219+0000","last_active":"2024-09-14T04:32:58.328796+0000","last_peered":"2024-09-14T04:32:58.328796+0000","last_clean":"2024-09-14T04:32:58.328796+0000","last_became_active":"2024-09-14T04:29:37.991324+0000","last_became_peered":"2024-09-14T04:29:37.991324+0000","last_unstale":"2024-09-14T04:32:58.328796+0000","last_undegraded":"2024-09-14T04:32:58.328796+0000","last_fullsized":"2024-09-14T04:32:58.328796+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:08.893407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.330039+0000","last_change":"2024-09-14T04:32:52.817519+0000","last_active":"2024-09-14T04:32:58.330039+0000","last_peered":"2024-09-14T04:32:58.330039+0000","last_clean":"2024-09-14T04:32:58.330039+0000","last_became_active":"2024-09-14T04:32:52.817207+0000","last_became_peered":"2024-09-14T04:32:52.817207+0000","last_unstale":"2024-09-14T04:32:58.330039+0000","last_undegraded":"2024-09-14T04:32:58.330039+0000","last_fullsized":"2024-09-14T04:32:58.330039+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:43.967076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":383,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328326+0000","last_change":"2024-09-14T04:32:55.318369+0000","last_active":"2024-09-14T04:32:58.328326+0000","last_peered":"2024-09-14T04:32:58.328326+0000","last_clean":"2024-09-14T04:32:58.328326+0000","last_became_active":"2024-09-14T04:32:55.317929+0000","last_became_peered":"2024-09-14T04:32:55.317929+0000","last_unstale":"2024-09-14T04:32:58.328326+0000","last_undegraded":"2024-09-14T04:32:58.328326+0000","last_fullsized":"2024-09-14T04:32:58.328326+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":126,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.323882+0000","last_change":"2024-09-14T04:30:04.283624+0000","last_active":"2024-09-14T04:32:58.323882+0000","last_peered":"2024-09-14T04:32:58.323882+0000","last_clean":"2024-09-14T04:32:58.323882+0000","last_became_active":"2024-09-14T04:30:04.283298+0000","last_became_peered":"2024-09-14T04:30:04.283298+0000","last_unstale":"2024-09-14T04:32:58.323882+0000","last_undegraded":"2024-09-14T04:32:58.323882+0000","last_fullsized":"2024-09-14T04:32:58.323882+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:32.022169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":943,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.337773+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:32:59.337773+0000","last_peered":"2024-09-14T04:32:59.337773+0000","last_clean":"2024-09-14T04:32:59.337773+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:32:59.337773+0000","last_undegraded":"2024-09-14T04:32:59.337773+0000","last_fullsized":"2024-09-14T04:32:59.337773+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":719,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339966+0000","last_change":"2024-09-14T04:30:04.285113+0000","last_active":"2024-09-14T04:32:59.339966+0000","last_peered":"2024-09-14T04:32:59.339966+0000","last_clean":"2024-09-14T04:32:59.339966+0000","last_became_active":"2024-09-14T04:30:04.283783+0000","last_became_peered":"2024-09-14T04:30:04.283783+0000","last_unstale":"2024-09-14T04:32:59.339966+0000","last_undegraded":"2024-09-14T04:32:59.339966+0000","last_fullsized":"2024-09-14T04:32:59.339966+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:13.747229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":813,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324522+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:32:58.324522+0000","last_peered":"2024-09-14T04:32:58.324522+0000","last_clean":"2024-09-14T04:32:58.324522+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:32:58.324522+0000","last_undegraded":"2024-09-14T04:32:58.324522+0000","last_fullsized":"2024-09-14T04:32:58.324522+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":512,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338257+0000","last_change":"2024-09-14T04:30:04.285510+0000","last_active":"2024-09-14T04:32:59.338257+0000","last_peered":"2024-09-14T04:32:59.338257+0000","last_clean":"2024-09-14T04:32:59.338257+0000","last_became_active":"2024-09-14T04:30:04.285170+0000","last_became_peered":"2024-09-14T04:30:04.285170+0000","last_unstale":"2024-09-14T04:32:59.338257+0000","last_undegraded":"2024-09-14T04:32:59.338257+0000","last_fullsized":"2024-09-14T04:32:59.338257+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:32.780039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329269+0000","last_change":"2024-09-14T04:32:52.815265+0000","last_active":"2024-09-14T04:32:58.329269+0000","last_peered":"2024-09-14T04:32:58.329269+0000","last_clean":"2024-09-14T04:32:58.329269+0000","last_became_active":"2024-09-14T04:32:52.814764+0000","last_became_peered":"2024-09-14T04:32:52.814764+0000","last_unstale":"2024-09-14T04:32:58.329269+0000","last_undegraded":"2024-09-14T04:32:58.329269+0000","last_fullsized":"2024-09-14T04:32:58.329269+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:59.683211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":471,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339482+0000","last_change":"2024-09-14T04:30:04.287957+0000","last_active":"2024-09-14T04:32:59.339482+0000","last_peered":"2024-09-14T04:32:59.339482+0000","last_clean":"2024-09-14T04:32:59.339482+0000","last_became_active":"2024-09-14T04:30:04.287313+0000","last_became_peered":"2024-09-14T04:30:04.287313+0000","last_unstale":"2024-09-14T04:32:59.339482+0000","last_undegraded":"2024-09-14T04:32:59.339482+0000","last_fullsized":"2024-09-14T04:32:59.339482+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:25.063506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":690,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338970+0000","last_change":"2024-09-14T04:29:40.285341+0000","last_active":"2024-09-14T04:32:59.338970+0000","last_peered":"2024-09-14T04:32:59.338970+0000","last_clean":"2024-09-14T04:32:59.338970+0000","last_became_active":"2024-09-14T04:29:40.284955+0000","last_became_peered":"2024-09-14T04:29:40.284955+0000","last_unstale":"2024-09-14T04:32:59.338970+0000","last_undegraded":"2024-09-14T04:32:59.338970+0000","last_fullsized":"2024-09-14T04:32:59.338970+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:18:51.087521+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328278+0000","last_change":"2024-09-14T04:32:57.968797+0000","last_active":"2024-09-14T04:32:58.328278+0000","last_peered":"2024-09-14T04:32:58.328278+0000","last_clean":"2024-09-14T04:32:58.328278+0000","last_became_active":"2024-09-14T04:32:52.810067+0000","last_became_peered":"2024-09-14T04:32:52.810067+0000","last_unstale":"2024-09-14T04:32:58.328278+0000","last_undegraded":"2024-09-14T04:32:58.328278+0000","last_fullsized":"2024-09-14T04:32:58.328278+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:19.561540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":577,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325368+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:32:58.325368+0000","last_peered":"2024-09-14T04:32:58.325368+0000","last_clean":"2024-09-14T04:32:58.325368+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:32:58.325368+0000","last_undegraded":"2024-09-14T04:32:58.325368+0000","last_fullsized":"2024-09-14T04:32:58.325368+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328555+0000","last_change":"2024-09-14T04:32:52.807897+0000","last_active":"2024-09-14T04:32:58.328555+0000","last_peered":"2024-09-14T04:32:58.328555+0000","last_clean":"2024-09-14T04:32:58.328555+0000","last_became_active":"2024-09-14T04:32:52.806944+0000","last_became_peered":"2024-09-14T04:32:52.806944+0000","last_unstale":"2024-09-14T04:32:58.328555+0000","last_undegraded":"2024-09-14T04:32:58.328555+0000","last_fullsized":"2024-09-14T04:32:58.328555+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:23.347741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":941,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329440+0000","last_change":"2024-09-14T04:32:55.315948+0000","last_active":"2024-09-14T04:32:58.329440+0000","last_peered":"2024-09-14T04:32:58.329440+0000","last_clean":"2024-09-14T04:32:58.329440+0000","last_became_active":"2024-09-14T04:32:55.315594+0000","last_became_peered":"2024-09-14T04:32:55.315594+0000","last_unstale":"2024-09-14T04:32:58.329440+0000","last_undegraded":"2024-09-14T04:32:58.329440+0000","last_fullsized":"2024-09-14T04:32:58.329440+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:59.189391+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":560,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.330767+0000","last_change":"2024-09-14T04:29:37.992555+0000","last_active":"2024-09-14T04:32:59.330767+0000","last_peered":"2024-09-14T04:32:59.330767+0000","last_clean":"2024-09-14T04:32:59.330767+0000","last_became_active":"2024-09-14T04:29:37.992147+0000","last_became_peered":"2024-09-14T04:29:37.992147+0000","last_unstale":"2024-09-14T04:32:59.330767+0000","last_undegraded":"2024-09-14T04:32:59.330767+0000","last_fullsized":"2024-09-14T04:32:59.330767+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:26:44.313121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":872,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338492+0000","last_change":"2024-09-14T04:30:04.287689+0000","last_active":"2024-09-14T04:32:59.338492+0000","last_peered":"2024-09-14T04:32:59.338492+0000","last_clean":"2024-09-14T04:32:59.338492+0000","last_became_active":"2024-09-14T04:30:04.287280+0000","last_became_peered":"2024-09-14T04:30:04.287280+0000","last_unstale":"2024-09-14T04:32:59.338492+0000","last_undegraded":"2024-09-14T04:32:59.338492+0000","last_fullsized":"2024-09-14T04:32:59.338492+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:19.006536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325527+0000","last_change":"2024-09-14T04:30:04.294612+0000","last_active":"2024-09-14T04:32:58.325527+0000","last_peered":"2024-09-14T04:32:58.325527+0000","last_clean":"2024-09-14T04:32:58.325527+0000","last_became_active":"2024-09-14T04:30:04.294288+0000","last_became_peered":"2024-09-14T04:30:04.294288+0000","last_unstale":"2024-09-14T04:32:58.325527+0000","last_undegraded":"2024-09-14T04:32:58.325527+0000","last_fullsized":"2024-09-14T04:32:58.325527+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:58.493527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":871,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329782+0000","last_change":"2024-09-14T04:28:22.699361+0000","last_active":"2024-09-14T04:32:58.329782+0000","last_peered":"2024-09-14T04:32:58.329782+0000","last_clean":"2024-09-14T04:32:58.329782+0000","last_became_active":"2024-09-14T04:28:22.699020+0000","last_became_peered":"2024-09-14T04:28:22.699020+0000","last_unstale":"2024-09-14T04:32:58.329782+0000","last_undegraded":"2024-09-14T04:32:58.329782+0000","last_fullsized":"2024-09-14T04:32:58.329782+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:07.313428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":935,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338026+0000","last_change":"2024-09-14T04:30:04.279568+0000","last_active":"2024-09-14T04:32:59.338026+0000","last_peered":"2024-09-14T04:32:59.338026+0000","last_clean":"2024-09-14T04:32:59.338026+0000","last_became_active":"2024-09-14T04:30:04.279227+0000","last_became_peered":"2024-09-14T04:30:04.279227+0000","last_unstale":"2024-09-14T04:32:59.338026+0000","last_undegraded":"2024-09-14T04:32:59.338026+0000","last_fullsized":"2024-09-14T04:32:59.338026+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:46.120838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325574+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:32:58.325574+0000","last_peered":"2024-09-14T04:32:58.325574+0000","last_clean":"2024-09-14T04:32:58.325574+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:32:58.325574+0000","last_undegraded":"2024-09-14T04:32:58.325574+0000","last_fullsized":"2024-09-14T04:32:58.325574+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":717,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.343585+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:32:59.343585+0000","last_peered":"2024-09-14T04:32:59.343585+0000","last_clean":"2024-09-14T04:32:59.343585+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:32:59.343585+0000","last_undegraded":"2024-09-14T04:32:59.343585+0000","last_fullsized":"2024-09-14T04:32:59.343585+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":888,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324721+0000","last_change":"2024-09-14T04:32:55.317272+0000","last_active":"2024-09-14T04:32:58.324721+0000","last_peered":"2024-09-14T04:32:58.324721+0000","last_clean":"2024-09-14T04:32:58.324721+0000","last_became_active":"2024-09-14T04:32:55.316545+0000","last_became_peered":"2024-09-14T04:32:55.316545+0000","last_unstale":"2024-09-14T04:32:58.324721+0000","last_undegraded":"2024-09-14T04:32:58.324721+0000","last_fullsized":"2024-09-14T04:32:58.324721+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:24.194356+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":121,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325284+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:32:58.325284+0000","last_peered":"2024-09-14T04:32:58.325284+0000","last_clean":"2024-09-14T04:32:58.325284+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:32:58.325284+0000","last_undegraded":"2024-09-14T04:32:58.325284+0000","last_fullsized":"2024-09-14T04:32:58.325284+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":805,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329726+0000","last_change":"2024-09-14T04:30:04.291109+0000","last_active":"2024-09-14T04:32:58.329726+0000","last_peered":"2024-09-14T04:32:58.329726+0000","last_clean":"2024-09-14T04:32:58.329726+0000","last_became_active":"2024-09-14T04:30:04.290804+0000","last_became_peered":"2024-09-14T04:30:04.290804+0000","last_unstale":"2024-09-14T04:32:58.329726+0000","last_undegraded":"2024-09-14T04:32:58.329726+0000","last_fullsized":"2024-09-14T04:32:58.329726+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:05.742866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":117,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324089+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:32:58.324089+0000","last_peered":"2024-09-14T04:32:58.324089+0000","last_clean":"2024-09-14T04:32:58.324089+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:32:58.324089+0000","last_undegraded":"2024-09-14T04:32:58.324089+0000","last_fullsized":"2024-09-14T04:32:58.324089+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1230,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.331040+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:32:59.331040+0000","last_peered":"2024-09-14T04:32:59.331040+0000","last_clean":"2024-09-14T04:32:59.331040+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:32:59.331040+0000","last_undegraded":"2024-09-14T04:32:59.331040+0000","last_fullsized":"2024-09-14T04:32:59.331040+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":720,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339782+0000","last_change":"2024-09-14T04:30:04.305324+0000","last_active":"2024-09-14T04:32:59.339782+0000","last_peered":"2024-09-14T04:32:59.339782+0000","last_clean":"2024-09-14T04:32:59.339782+0000","last_became_active":"2024-09-14T04:30:04.289343+0000","last_became_peered":"2024-09-14T04:30:04.289343+0000","last_unstale":"2024-09-14T04:32:59.339782+0000","last_undegraded":"2024-09-14T04:32:59.339782+0000","last_fullsized":"2024-09-14T04:32:59.339782+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:41.011672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329521+0000","last_change":"2024-09-14T04:29:37.991840+0000","last_active":"2024-09-14T04:32:58.329521+0000","last_peered":"2024-09-14T04:32:58.329521+0000","last_clean":"2024-09-14T04:32:58.329521+0000","last_became_active":"2024-09-14T04:29:37.991503+0000","last_became_peered":"2024-09-14T04:29:37.991503+0000","last_unstale":"2024-09-14T04:32:58.329521+0000","last_undegraded":"2024-09-14T04:32:58.329521+0000","last_fullsized":"2024-09-14T04:32:58.329521+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:22.858456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":543,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325225+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:32:58.325225+0000","last_peered":"2024-09-14T04:32:58.325225+0000","last_clean":"2024-09-14T04:32:58.325225+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:32:58.325225+0000","last_undegraded":"2024-09-14T04:32:58.325225+0000","last_fullsized":"2024-09-14T04:32:58.325225+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":597,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324590+0000","last_change":"2024-09-14T04:30:04.291353+0000","last_active":"2024-09-14T04:32:58.324590+0000","last_peered":"2024-09-14T04:32:58.324590+0000","last_clean":"2024-09-14T04:32:58.324590+0000","last_became_active":"2024-09-14T04:30:04.290907+0000","last_became_peered":"2024-09-14T04:30:04.290907+0000","last_unstale":"2024-09-14T04:32:58.324590+0000","last_undegraded":"2024-09-14T04:32:58.324590+0000","last_fullsized":"2024-09-14T04:32:58.324590+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:17.989312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":607,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325167+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:32:58.325167+0000","last_peered":"2024-09-14T04:32:58.325167+0000","last_clean":"2024-09-14T04:32:58.325167+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:32:58.325167+0000","last_undegraded":"2024-09-14T04:32:58.325167+0000","last_fullsized":"2024-09-14T04:32:58.325167+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.328818+0000","last_change":"2024-09-14T04:32:52.811129+0000","last_active":"2024-09-14T04:32:58.328818+0000","last_peered":"2024-09-14T04:32:58.328818+0000","last_clean":"2024-09-14T04:32:58.328818+0000","last_became_active":"2024-09-14T04:32:52.810519+0000","last_became_peered":"2024-09-14T04:32:52.810519+0000","last_unstale":"2024-09-14T04:32:58.328818+0000","last_undegraded":"2024-09-14T04:32:58.328818+0000","last_fullsized":"2024-09-14T04:32:58.328818+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:34.831570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":123,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325504+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:32:58.325504+0000","last_peered":"2024-09-14T04:32:58.325504+0000","last_clean":"2024-09-14T04:32:58.325504+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:32:58.325504+0000","last_undegraded":"2024-09-14T04:32:58.325504+0000","last_fullsized":"2024-09-14T04:32:58.325504+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":1044,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325163+0000","last_change":"2024-09-14T04:32:55.317954+0000","last_active":"2024-09-14T04:32:58.325163+0000","last_peered":"2024-09-14T04:32:58.325163+0000","last_clean":"2024-09-14T04:32:58.325163+0000","last_became_active":"2024-09-14T04:32:55.317252+0000","last_became_peered":"2024-09-14T04:32:55.317252+0000","last_unstale":"2024-09-14T04:32:58.325163+0000","last_undegraded":"2024-09-14T04:32:58.325163+0000","last_fullsized":"2024-09-14T04:32:58.325163+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:02.900907+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":408,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324623+0000","last_change":"2024-09-14T04:30:04.283436+0000","last_active":"2024-09-14T04:32:58.324623+0000","last_peered":"2024-09-14T04:32:58.324623+0000","last_clean":"2024-09-14T04:32:58.324623+0000","last_became_active":"2024-09-14T04:30:04.283122+0000","last_became_peered":"2024-09-14T04:30:04.283122+0000","last_unstale":"2024-09-14T04:32:58.324623+0000","last_undegraded":"2024-09-14T04:32:58.324623+0000","last_fullsized":"2024-09-14T04:32:58.324623+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.646724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":126,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324074+0000","last_change":"2024-09-14T04:28:19.257505+0000","last_active":"2024-09-14T04:32:58.324074+0000","last_peered":"2024-09-14T04:32:58.324074+0000","last_clean":"2024-09-14T04:32:58.324074+0000","last_became_active":"2024-09-14T04:28:19.257193+0000","last_became_peered":"2024-09-14T04:28:19.257193+0000","last_unstale":"2024-09-14T04:32:58.324074+0000","last_undegraded":"2024-09-14T04:32:58.324074+0000","last_fullsized":"2024-09-14T04:32:58.324074+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:03:33.146736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":704,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339578+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:32:59.339578+0000","last_peered":"2024-09-14T04:32:59.339578+0000","last_clean":"2024-09-14T04:32:59.339578+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:32:59.339578+0000","last_undegraded":"2024-09-14T04:32:59.339578+0000","last_fullsized":"2024-09-14T04:32:59.339578+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":822,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.330101+0000","last_change":"2024-09-14T04:32:52.817482+0000","last_active":"2024-09-14T04:32:59.330101+0000","last_peered":"2024-09-14T04:32:59.330101+0000","last_clean":"2024-09-14T04:32:59.330101+0000","last_became_active":"2024-09-14T04:32:52.817095+0000","last_became_peered":"2024-09-14T04:32:52.817095+0000","last_unstale":"2024-09-14T04:32:59.330101+0000","last_undegraded":"2024-09-14T04:32:59.330101+0000","last_fullsized":"2024-09-14T04:32:59.330101+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:31.086628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329593+0000","last_change":"2024-09-14T04:30:04.283086+0000","last_active":"2024-09-14T04:32:58.329593+0000","last_peered":"2024-09-14T04:32:58.329593+0000","last_clean":"2024-09-14T04:32:58.329593+0000","last_became_active":"2024-09-14T04:30:04.282704+0000","last_became_peered":"2024-09-14T04:30:04.282704+0000","last_unstale":"2024-09-14T04:32:58.329593+0000","last_undegraded":"2024-09-14T04:32:58.329593+0000","last_fullsized":"2024-09-14T04:32:58.329593+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:03.488767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":48,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.323897+0000","last_change":"2024-09-14T04:30:04.283463+0000","last_active":"2024-09-14T04:32:58.323897+0000","last_peered":"2024-09-14T04:32:58.323897+0000","last_clean":"2024-09-14T04:32:58.323897+0000","last_became_active":"2024-09-14T04:30:04.283157+0000","last_became_peered":"2024-09-14T04:30:04.283157+0000","last_unstale":"2024-09-14T04:32:58.323897+0000","last_undegraded":"2024-09-14T04:32:58.323897+0000","last_fullsized":"2024-09-14T04:32:58.323897+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:02:11.942507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":728,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339690+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:32:59.339690+0000","last_peered":"2024-09-14T04:32:59.339690+0000","last_clean":"2024-09-14T04:32:59.339690+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:32:59.339690+0000","last_undegraded":"2024-09-14T04:32:59.339690+0000","last_fullsized":"2024-09-14T04:32:59.339690+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":753,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.330388+0000","last_change":"2024-09-14T04:30:04.298473+0000","last_active":"2024-09-14T04:32:59.330388+0000","last_peered":"2024-09-14T04:32:59.330388+0000","last_clean":"2024-09-14T04:32:59.330388+0000","last_became_active":"2024-09-14T04:30:04.295176+0000","last_became_peered":"2024-09-14T04:30:04.295176+0000","last_unstale":"2024-09-14T04:32:59.330388+0000","last_undegraded":"2024-09-14T04:32:59.330388+0000","last_fullsized":"2024-09-14T04:32:59.330388+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:02:30.762732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":287,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324807+0000","last_change":"2024-09-14T04:32:52.818866+0000","last_active":"2024-09-14T04:32:58.324807+0000","last_peered":"2024-09-14T04:32:58.324807+0000","last_clean":"2024-09-14T04:32:58.324807+0000","last_became_active":"2024-09-14T04:32:52.817822+0000","last_became_peered":"2024-09-14T04:32:52.817822+0000","last_unstale":"2024-09-14T04:32:58.324807+0000","last_undegraded":"2024-09-14T04:32:58.324807+0000","last_fullsized":"2024-09-14T04:32:58.324807+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:56:44.935438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":155,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324383+0000","last_change":"2024-09-14T04:30:04.295082+0000","last_active":"2024-09-14T04:32:58.324383+0000","last_peered":"2024-09-14T04:32:58.324383+0000","last_clean":"2024-09-14T04:32:58.324383+0000","last_became_active":"2024-09-14T04:30:04.293846+0000","last_became_peered":"2024-09-14T04:30:04.293846+0000","last_unstale":"2024-09-14T04:32:58.324383+0000","last_undegraded":"2024-09-14T04:32:58.324383+0000","last_fullsized":"2024-09-14T04:32:58.324383+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:05.873391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":343,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338147+0000","last_change":"2024-09-14T04:30:04.293298+0000","last_active":"2024-09-14T04:32:59.338147+0000","last_peered":"2024-09-14T04:32:59.338147+0000","last_clean":"2024-09-14T04:32:59.338147+0000","last_became_active":"2024-09-14T04:30:04.292876+0000","last_became_peered":"2024-09-14T04:30:04.292876+0000","last_unstale":"2024-09-14T04:32:59.338147+0000","last_undegraded":"2024-09-14T04:32:59.338147+0000","last_fullsized":"2024-09-14T04:32:59.338147+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:13.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":731,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.343491+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:32:59.343491+0000","last_peered":"2024-09-14T04:32:59.343491+0000","last_clean":"2024-09-14T04:32:59.343491+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:32:59.343491+0000","last_undegraded":"2024-09-14T04:32:59.343491+0000","last_fullsized":"2024-09-14T04:32:59.343491+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":167,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324380+0000","last_change":"2024-09-14T04:30:04.296875+0000","last_active":"2024-09-14T04:32:58.324380+0000","last_peered":"2024-09-14T04:32:58.324380+0000","last_clean":"2024-09-14T04:32:58.324380+0000","last_became_active":"2024-09-14T04:30:04.296576+0000","last_became_peered":"2024-09-14T04:30:04.296576+0000","last_unstale":"2024-09-14T04:32:58.324380+0000","last_undegraded":"2024-09-14T04:32:58.324380+0000","last_fullsized":"2024-09-14T04:32:58.324380+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:40.996256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":278,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324561+0000","last_change":"2024-09-14T04:32:52.824597+0000","last_active":"2024-09-14T04:32:58.324561+0000","last_peered":"2024-09-14T04:32:58.324561+0000","last_clean":"2024-09-14T04:32:58.324561+0000","last_became_active":"2024-09-14T04:32:52.818458+0000","last_became_peered":"2024-09-14T04:32:52.818458+0000","last_unstale":"2024-09-14T04:32:58.324561+0000","last_undegraded":"2024-09-14T04:32:58.324561+0000","last_fullsized":"2024-09-14T04:32:58.324561+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:02.830142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":158,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.325569+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:32:58.325569+0000","last_peered":"2024-09-14T04:32:58.325569+0000","last_clean":"2024-09-14T04:32:58.325569+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:32:58.325569+0000","last_undegraded":"2024-09-14T04:32:58.325569+0000","last_fullsized":"2024-09-14T04:32:58.325569+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":456,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324627+0000","last_change":"2024-09-14T04:32:52.822812+0000","last_active":"2024-09-14T04:32:58.324627+0000","last_peered":"2024-09-14T04:32:58.324627+0000","last_clean":"2024-09-14T04:32:58.324627+0000","last_became_active":"2024-09-14T04:32:52.817376+0000","last_became_peered":"2024-09-14T04:32:52.817376+0000","last_unstale":"2024-09-14T04:32:58.324627+0000","last_undegraded":"2024-09-14T04:32:58.324627+0000","last_fullsized":"2024-09-14T04:32:58.324627+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:38.165374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":452,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.338347+0000","last_change":"2024-09-14T04:29:38.010269+0000","last_active":"2024-09-14T04:32:59.338347+0000","last_peered":"2024-09-14T04:32:59.338347+0000","last_clean":"2024-09-14T04:32:59.338347+0000","last_became_active":"2024-09-14T04:29:38.009875+0000","last_became_peered":"2024-09-14T04:29:38.009875+0000","last_unstale":"2024-09-14T04:32:59.338347+0000","last_undegraded":"2024-09-14T04:32:59.338347+0000","last_fullsized":"2024-09-14T04:32:59.338347+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:38.314967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":18,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.329507+0000","last_change":"2024-09-14T04:32:52.808518+0000","last_active":"2024-09-14T04:32:58.329507+0000","last_peered":"2024-09-14T04:32:58.329507+0000","last_clean":"2024-09-14T04:32:58.329507+0000","last_became_active":"2024-09-14T04:32:52.807430+0000","last_became_peered":"2024-09-14T04:32:52.807430+0000","last_unstale":"2024-09-14T04:32:58.329507+0000","last_undegraded":"2024-09-14T04:32:58.329507+0000","last_fullsized":"2024-09-14T04:32:58.329507+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:24.095929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":548,"reported_epoch":545,"state":"active+clean","last_fresh":"2024-09-14T04:32:58.324326+0000","last_change":"2024-09-14T04:32:52.818107+0000","last_active":"2024-09-14T04:32:58.324326+0000","last_peered":"2024-09-14T04:32:58.324326+0000","last_clean":"2024-09-14T04:32:58.324326+0000","last_became_active":"2024-09-14T04:32:52.817622+0000","last_became_peered":"2024-09-14T04:32:52.817622+0000","last_unstale":"2024-09-14T04:32:58.324326+0000","last_undegraded":"2024-09-14T04:32:58.324326+0000","last_fullsized":"2024-09-14T04:32:58.324326+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:14.162666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":765,"reported_epoch":546,"state":"active+clean","last_fresh":"2024-09-14T04:32:59.339680+0000","last_change":"2024-09-14T04:29:38.016663+0000","last_active":"2024-09-14T04:32:59.339680+0000","last_peered":"2024-09-14T04:32:59.339680+0000","last_clean":"2024-09-14T04:32:59.339680+0000","last_became_active":"2024-09-14T04:29:38.007007+0000","last_became_peered":"2024-09-14T04:29:38.007007+0000","last_unstale":"2024-09-14T04:32:59.339680+0000","last_undegraded":"2024-09-14T04:32:59.339680+0000","last_fullsized":"2024-09-14T04:32:59.339680+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:55:55.070473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":22506,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":399240,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":100,"num_write":25,"num_write_kb":292,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2443447,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201909,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1475664,"kb_used_data":7872,"kb_used_omap":1,"kb_used_meta":1467774,"kb_avail":92269488,"statfs":{"total":95995035648,"available":94483955712,"internally_reserved":0,"allocated":8060928,"data_stored":4482938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1503001035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463547,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1194960,"kb_used_data":8444,"kb_used_omap":1,"kb_used_meta":1186494,"kb_avail":92550192,"statfs":{"total":95995035648,"available":94771396608,"internally_reserved":0,"allocated":8646656,"data_stored":5073241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1214970311},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834220,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":621248,"kb_used_data":8112,"kb_used_omap":5,"kb_used_meta":613114,"kb_avail":93123904,"statfs":{"total":95995035648,"available":95358877696,"internally_reserved":0,"allocated":8306688,"data_stored":4741044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":627828797},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986819,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1730564,"kb_used_data":7864,"kb_used_omap":1,"kb_used_meta":1722686,"kb_avail":92014588,"statfs":{"total":95995035648,"available":94222938112,"internally_reserved":0,"allocated":8052736,"data_stored":4482595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1764030918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748848,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1270200,"kb_used_data":8428,"kb_used_omap":4,"kb_used_meta":1261755,"kb_avail":92474952,"statfs":{"total":95995035648,"available":94694350848,"internally_reserved":0,"allocated":8630272,"data_stored":5068436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1292037676},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":1275605287069,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":463856468391,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342506,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000192,"kb_used_data":8176,"kb_used_omap":10,"kb_used_meta":991989,"kb_avail":92744960,"statfs":{"total":95995035648,"available":94970839040,"internally_reserved":0,"allocated":8372224,"data_stored":4806281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":1015796946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:33:04.780 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:33:04.836 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:04.938 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:04.962 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:04.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:05.064 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:05.087 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:05.189 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:05.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:05.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:05.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:05.340 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:05.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:05.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:05.468 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:05.509 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T04:33:05.509 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-14T04:33:05.509 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-14T04:33:05.515 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-14T04:33:05.515 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T04:33:05.516 DEBUG:teuthology.orchestra.run.smithi001:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T04:33:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.545 INFO:teuthology.orchestra.run.smithi001.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T04:33:05.545 DEBUG:teuthology.orchestra.run.smithi001:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T04:33:05.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:05.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:05 smithi012 ceph-mon[101180]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:05 smithi012 ceph-mon[101180]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:33:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[112165]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[112165]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[107399]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[107399]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 470/1047 objects degraded (44.890%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:05.606 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:05.608 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:05.611 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T04:33:05.611 DEBUG:teuthology.orchestra.run.smithi001:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T04:33:05.666 INFO:tasks.workunit:timeout=3h 2024-09-14T04:33:05.666 INFO:tasks.workunit:cleanup=True 2024-09-14T04:33:05.666 DEBUG:teuthology.orchestra.run.smithi001:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-14T04:33:05.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:05.727 INFO:tasks.workunit.client.0.smithi001.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T04:33:05.739 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:05.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:05.867 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:05.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:05.994 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:05.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:06.096 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:06.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:06.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:06.248 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:06.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:06.377 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:06.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:06 smithi001 ceph-mon[107399]: pgmap v1733: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:06.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:06 smithi001 ceph-mon[112165]: pgmap v1733: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:06.478 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:06.489 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.509 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:06.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:06.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:06.637 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:06.739 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:06.761 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:06.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:06 smithi012 ceph-mon[101180]: pgmap v1733: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:06.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:06.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:06.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:06.931 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:06] "GET /metrics HTTP/1.1" 200 34190 "" "Prometheus/2.51.0" 2024-09-14T04:33:06.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:07.015 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:07.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:07.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:07.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:07.272 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:07.374 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:07.398 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:07.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:07.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:07.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:07.654 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:07.756 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:07.779 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:07.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:07.903 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:07.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:07.997 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:08.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:08.033 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:08.135 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:08.157 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:08.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:08.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:08.286 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:08.388 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:08.410 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:08.512 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.536 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:08.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:08.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:08.664 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:08.766 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:08.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.790 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:08.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:08.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:08.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:08.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:08 smithi001 ceph-mon[107399]: pgmap v1734: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:08.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:08 smithi001 ceph-mon[112165]: pgmap v1734: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:09.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:09.047 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:09.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:09.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:08 smithi012 ceph-mon[101180]: pgmap v1734: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:09.148 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:09.171 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:09.273 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:09.294 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:09.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:09.419 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:09.520 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:09.543 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:09.645 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:09.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:09.768 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:09.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.793 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:09.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:09.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:09.916 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:09.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:10.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:10.040 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:10.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:10.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:10.164 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:10.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:10.287 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:10.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:10.411 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:10.514 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:10.538 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:10.640 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:10.662 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:10.763 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:10.786 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:10.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:10.912 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:10.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:10.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:10 smithi001 ceph-mon[112165]: pgmap v1735: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:10.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:10 smithi001 ceph-mon[107399]: pgmap v1735: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:11.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:11.036 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:11.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:10 smithi012 ceph-mon[101180]: pgmap v1735: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:11.138 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:11.161 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:11.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:11.288 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:11.391 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:11.413 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:11.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:11.542 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:11.644 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:11.667 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:11.769 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:11.791 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:11.893 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:11.915 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:11.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:12.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:12.045 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:12.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:12.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:12.173 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:12.275 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:12.300 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:12.402 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:12.426 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:12.528 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:12.552 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:12.654 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.677 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:12.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:12.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:12.800 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:12.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:12.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:12.930 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:12.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:12.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:12 smithi001 ceph-mon[107399]: pgmap v1736: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:12 smithi001 ceph-mon[112165]: pgmap v1736: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:13.031 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:13.053 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:12 smithi012 ceph-mon[101180]: pgmap v1736: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:13.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:13.182 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:13.284 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:13.307 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:13.409 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:13.431 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:13.534 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:13.561 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:13.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:13.593 INFO:tasks.workunit.client.0.smithi001.stderr:Updating files: 90% (10943/12129) Updating files: 91% (11038/12129) Updating files: 92% (11159/12129) Updating files: 93% (11280/12129) Updating files: 94% (11402/12129) Updating files: 95% (11523/12129) Updating files: 96% (11644/12129) Updating files: 97% (11766/12129) Updating files: 98% (11887/12129) Updating files: 99% (12008/12129) Updating files: 100% (12129/12129) Updating files: 100% (12129/12129), done. 2024-09-14T04:33:13.610 DEBUG:teuthology.orchestra.run.smithi001:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T04:33:13.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:13.690 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:13.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:13.814 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:13.915 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:13.937 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:13.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:13.973 INFO:tasks.workunit.client.0.smithi001.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T04:33:13.976 INFO:tasks.workunit.client.0.smithi001.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T04:33:13.976 INFO:tasks.workunit.client.0.smithi001.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T04:33:14.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:14.065 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:14.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:14.167 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:14.189 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:14.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:14.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:14.313 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:14.415 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:14.439 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:14.541 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:14.563 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:14.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:14.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.693 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:14.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:14.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:14.821 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:14.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:14.946 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:14.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:14.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:14 smithi001 ceph-mon[107399]: pgmap v1737: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:14.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:14 smithi001 ceph-mon[112165]: pgmap v1737: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:15.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:15.075 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:15.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:14 smithi012 ceph-mon[101180]: pgmap v1737: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:15.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:15.203 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:15.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:15.333 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:15.435 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:15.462 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:15.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:15.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:15.592 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:15.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:15.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:15.720 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:15.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:15.849 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:15.951 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:15.979 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:15.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:16.081 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:16.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:16.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:16.237 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:16.340 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:16.367 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:16.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:16.496 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:16.598 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:16.627 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:16.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:16.756 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:16.858 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:16.885 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:16.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:16.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:16 smithi001 ceph-mon[107399]: pgmap v1738: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:16.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:16 smithi001 ceph-mon[112165]: pgmap v1738: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:16.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:16] "GET /metrics HTTP/1.1" 200 34190 "" "Prometheus/2.51.0" 2024-09-14T04:33:16.987 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:17.015 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:17.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:16 smithi012 ceph-mon[101180]: pgmap v1738: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:17.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:17.143 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:17.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:17.272 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:17.373 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:17.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:17.503 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:17.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:17.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:17.659 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:17.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:17.788 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:17.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:17.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:17.917 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:17.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:18.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:18.044 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:18.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:18.146 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:18.174 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:18.276 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:18.303 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:18.405 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:18.432 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:18.534 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:18.561 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:18.663 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:18.691 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:18.793 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:18.822 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:18.923 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:18.948 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:18.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:18.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:18 smithi001 ceph-mon[107399]: pgmap v1739: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:18.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:18 smithi001 ceph-mon[112165]: pgmap v1739: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:19.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:19.078 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:19.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:18 smithi012 ceph-mon[101180]: pgmap v1739: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:19.179 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:19.206 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:19.308 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:19.334 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:19.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:19.463 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:19.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:19.565 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.593 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:19.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:19.694 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:19.724 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:19.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:19.853 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:19.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:19.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:20.082 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:20.184 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:20.211 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:20.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:20.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:20.339 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:20.441 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:20.468 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:20.570 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:20.598 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:20.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:20.729 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:20.831 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:20.854 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:20.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:20.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:20 smithi001 ceph-mon[107399]: pgmap v1740: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:20.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:20 smithi001 ceph-mon[112165]: pgmap v1740: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:20.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:20.984 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:20.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:21.085 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:21.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:20 smithi012 ceph-mon[101180]: pgmap v1740: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:21.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:21.109 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:21.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:21.238 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:21.339 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:21.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.363 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:21.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:21.464 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:21.487 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:21.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:21.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:21.616 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:21.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:21.638 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:21.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:21.743 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:21.845 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:21.868 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:21.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:21.996 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:21.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:21.998 INFO:tasks.workunit.client.0.smithi001.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T04:33:22.097 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:22.120 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:22.205 INFO:tasks.workunit.client.0.smithi001.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T04:33:22.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:22.248 INFO:tasks.workunit.client.0.smithi001.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T04:33:22.249 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:22.250 INFO:tasks.workunit.client.0.smithi001.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T04:33:22.250 INFO:tasks.workunit.client.0.smithi001.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T04:33:22.291 INFO:tasks.workunit.client.0.smithi001.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T04:33:22.323 DEBUG:teuthology.orchestra.run.smithi001:> set -ex 2024-09-14T04:33:22.323 DEBUG:teuthology.orchestra.run.smithi001:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T04:33:22.350 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:22.359 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-14T04:33:22.361 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-14T04:33:22.361 DEBUG:teuthology.orchestra.run.smithi001:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-14T04:33:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.373 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:22.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:22.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:22.501 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:22.603 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:22.623 INFO:tasks.workunit.client.0.smithi001.stdout:Running main() from gmock_main.cc 2024-09-14T04:33:22.623 INFO:tasks.workunit.client.0.smithi001.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-14T04:33:22.623 INFO:tasks.workunit.client.0.smithi001.stdout:[----------] Global test environment set-up. 2024-09-14T04:33:22.623 INFO:tasks.workunit.client.0.smithi001.stdout:[----------] 70 tests from TestClsRbd 2024-09-14T04:33:22.625 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:22.727 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:22.750 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:22.851 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:22.874 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:22.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:22.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: pgmap v1741: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:22.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: pgmap v1741: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:22.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:22.976 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:22.999 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:23.009 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:23.009 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: pgmap v1741: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:23.101 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:23.123 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:23.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:23.252 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:23.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:23.380 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:23.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:23.508 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:23.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:23.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:23.635 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:23.688 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:33:23.733 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-14T04:33:23.737 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:23.756 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_all_features (23 ms) 2024-09-14T04:33:23.756 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.copyup 2024-09-14T04:33:23.765 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:23.866 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:23.888 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:23.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:23.925 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[112165]: osdmap e548: 8 total, 8 up, 6 in 2024-09-14T04:33:23.925 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2772154583' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:23.925 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:23.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[107399]: osdmap e548: 8 total, 8 up, 6 in 2024-09-14T04:33:23.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2772154583' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:23.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:23 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:23.946 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.copyup (189 ms) 2024-09-14T04:33:23.946 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-14T04:33:23.982 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.sparse_copyup (36 ms) 2024-09-14T04:33:23.983 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-14T04:33:23.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:23.990 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-09-14T04:33:23.991 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-14T04:33:24.000 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-14T04:33:24.000 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-14T04:33:24.013 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:24.025 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.directory_methods (25 ms) 2024-09-14T04:33:24.025 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.create 2024-09-14T04:33:24.044 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.create (19 ms) 2024-09-14T04:33:24.044 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_features 2024-09-14T04:33:24.046 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-14T04:33:24.046 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-14T04:33:24.047 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-14T04:33:24.047 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-14T04:33:24.049 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-14T04:33:24.049 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-14T04:33:24.050 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-14T04:33:24.050 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-14T04:33:24.051 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-14T04:33:24.051 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-14T04:33:24.054 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-14T04:33:24.054 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.get_size 2024-09-14T04:33:24.058 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-14T04:33:24.058 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.set_size 2024-09-14T04:33:24.062 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-14T04:33:24.062 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-14T04:33:24.079 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-09-14T04:33:24.079 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-14T04:33:24.087 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-09-14T04:33:24.087 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-14T04:33:24.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:23 smithi012 ceph-mon[101180]: osdmap e548: 8 total, 8 up, 6 in 2024-09-14T04:33:24.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:23 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2772154583' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:24.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:23 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:33:24.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:24.138 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:24.167 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.parents_v1 (80 ms) 2024-09-14T04:33:24.167 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-14T04:33:24.195 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.parents_v2 (28 ms) 2024-09-14T04:33:24.195 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-14T04:33:24.216 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-09-14T04:33:24.217 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-14T04:33:24.220 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2024-09-14T04:33:24.220 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-14T04:33:24.228 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-09-14T04:33:24.228 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-14T04:33:24.230 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2024-09-14T04:33:24.230 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-14T04:33:24.235 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2024-09-14T04:33:24.235 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-14T04:33:24.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2024-09-14T04:33:24.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-14T04:33:24.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-14T04:33:24.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-14T04:33:24.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:24.244 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_snap_add (5 ms) 2024-09-14T04:33:24.244 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-14T04:33:24.248 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2024-09-14T04:33:24.248 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.flags 2024-09-14T04:33:24.254 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.flags (6 ms) 2024-09-14T04:33:24.254 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.metadata 2024-09-14T04:33:24.264 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:24.283 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.metadata (29 ms) 2024-09-14T04:33:24.283 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.set_features 2024-09-14T04:33:24.289 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-14T04:33:24.290 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror 2024-09-14T04:33:24.325 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2024-09-14T04:33:24.325 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-14T04:33:24.341 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2024-09-14T04:33:24.341 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-14T04:33:24.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:24.388 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:24.490 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:24.518 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:24.612 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:33:24.612 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:33:24.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:24.635 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:33:24.636 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:33:24.642 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:24.656 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:33:24.657 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:33:24.677 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:33:24.677 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:33:24.697 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:33:24.697 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:33:24.718 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:33:24.718 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:33:24.735 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[107399]: pgmap v1743: 125 pgs: 32 unknown, 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:24.735 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[107399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[107399]: osdmap e549: 8 total, 8 up, 6 in 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2197791631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[112165]: pgmap v1743: 125 pgs: 32 unknown, 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[112165]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[112165]: osdmap e549: 8 total, 8 up, 6 in 2024-09-14T04:33:24.736 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:24 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2197791631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:24.743 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:33:24.743 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:33:24.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:24.765 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:24.768 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:33:24.768 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:33:24.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:24.891 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:24.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:24.992 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:25.018 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:25.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:24 smithi012 ceph-mon[101180]: pgmap v1743: 125 pgs: 32 unknown, 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:24 smithi012 ceph-mon[101180]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:33:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:24 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi001-366220-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:33:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:24 smithi012 ceph-mon[101180]: osdmap e549: 8 total, 8 up, 6 in 2024-09-14T04:33:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:24 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2197791631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:25.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:25.142 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:25.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:25.266 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:25.327 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:25.368 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:25.393 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:25.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:25.517 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:25.619 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:25.645 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:25.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:25.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:25.769 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:25.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:25.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:25.893 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:25.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:25.995 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:26.020 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:26.058 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:25 smithi001 ceph-mon[107399]: osdmap e550: 8 total, 8 up, 6 in 2024-09-14T04:33:26.058 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:25 smithi001 ceph-mon[107399]: pgmap v1746: 125 pgs: 12 unknown, 113 active+clean; 396 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 837 KiB/s wr, 482 op/s 2024-09-14T04:33:26.058 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:25 smithi001 ceph-mon[112165]: osdmap e550: 8 total, 8 up, 6 in 2024-09-14T04:33:26.058 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:25 smithi001 ceph-mon[112165]: pgmap v1746: 125 pgs: 12 unknown, 113 active+clean; 396 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 837 KiB/s wr, 482 op/s 2024-09-14T04:33:26.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:25 smithi012 ceph-mon[101180]: osdmap e550: 8 total, 8 up, 6 in 2024-09-14T04:33:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:25 smithi012 ceph-mon[101180]: pgmap v1746: 125 pgs: 12 unknown, 113 active+clean; 396 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 837 KiB/s wr, 482 op/s 2024-09-14T04:33:26.122 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:26.147 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:26.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:26.275 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:26.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:26.401 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:26.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:26.527 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:26.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:26.656 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:26.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:26.780 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:26.793 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror_image_status (2452 ms) 2024-09-14T04:33:26.793 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-14T04:33:26.870 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror_image_map (76 ms) 2024-09-14T04:33:26.870 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-14T04:33:26.878 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2024-09-14T04:33:26.878 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-14T04:33:26.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:26.890 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2024-09-14T04:33:26.890 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-14T04:33:26.892 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-14T04:33:26.892 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-14T04:33:26.895 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2024-09-14T04:33:26.895 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-14T04:33:26.898 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-14T04:33:26.898 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-14T04:33:26.907 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_dir_rename (8 ms) 2024-09-14T04:33:26.907 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-14T04:33:26.907 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:26.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:26.910 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2024-09-14T04:33:26.911 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-14T04:33:26.915 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2024-09-14T04:33:26.915 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-14T04:33:26.918 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2024-09-14T04:33:26.918 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-14T04:33:26.921 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-14T04:33:26.921 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-14T04:33:26.924 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-14T04:33:26.924 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-14T04:33:26.927 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_image_clean (3 ms) 2024-09-14T04:33:26.927 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-14T04:33:26.929 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-09-14T04:33:26.930 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-14T04:33:26.933 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-14T04:33:26.933 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-14T04:33:26.935 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-14T04:33:26.935 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-14T04:33:26.937 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-14T04:33:26.937 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-14T04:33:26.939 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-14T04:33:26.939 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-14T04:33:26.942 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-14T04:33:26.942 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-14T04:33:26.944 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-09-14T04:33:26.944 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout:Expected equality of these values: 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout: 1U 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout: Which is: 1 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout: keys.size() 2024-09-14T04:33:26.946 INFO:tasks.workunit.client.0.smithi001.stdout: Which is: 3 2024-09-14T04:33:26.947 INFO:tasks.workunit.client.0.smithi001.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-14T04:33:26.947 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-14T04:33:26.950 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-09-14T04:33:26.950 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-14T04:33:26.965 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:26 smithi001 ceph-mon[107399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:33:26.965 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:26 smithi001 ceph-mon[112165]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:33:26.966 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:26] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.51.0" 2024-09-14T04:33:26.966 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-14T04:33:26.966 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-14T04:33:27.009 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:27.032 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:26 smithi012 ceph-mon[101180]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:33:27.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:27.158 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:27.174 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (208 ms) 2024-09-14T04:33:27.174 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-14T04:33:27.177 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-09-14T04:33:27.178 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-14T04:33:27.180 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2024-09-14T04:33:27.180 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-14T04:33:27.187 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-14T04:33:27.187 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.op_features 2024-09-14T04:33:27.193 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-09-14T04:33:27.193 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-14T04:33:27.213 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-09-14T04:33:27.213 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-14T04:33:27.221 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-14T04:33:27.221 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-14T04:33:27.232 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-09-14T04:33:27.232 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-14T04:33:27.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-14T04:33:27.239 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.migration 2024-09-14T04:33:27.248 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-14T04:33:27.248 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-14T04:33:27.257 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2024-09-14T04:33:27.257 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-14T04:33:27.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:27.281 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:27.383 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:27.405 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:27.507 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:27.530 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:27.632 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:27.654 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:27.749 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is set 2024-09-14T04:33:27.756 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:27.779 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:27.809 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:33:27.881 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:27.907 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:27.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:28.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:28.033 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4031974469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:27 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:27 smithi012 ceph-mon[101180]: pgmap v1747: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 847 KiB/s wr, 501 op/s 2024-09-14T04:33:28.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:33:28.161 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:33:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4031974469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[107399]: pgmap v1747: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 847 KiB/s wr, 501 op/s 2024-09-14T04:33:28.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4031974469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:28.222 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:27 smithi001 ceph-mon[112165]: pgmap v1747: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.6 MiB/s rd, 847 KiB/s wr, 501 op/s 2024-09-14T04:33:28.263 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:28.290 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:28.392 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:28.418 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:28.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:28.547 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:28.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:33:28.676 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:33:28.753 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1496 ms) 2024-09-14T04:33:28.753 INFO:tasks.workunit.client.0.smithi001.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-14T04:33:28.778 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:33:28.802 INFO:tasks.workunit.client.0.smithi001.stdout:[ OK ] TestClsRbd.sparsify (49 ms) 2024-09-14T04:33:28.803 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:33:28.905 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:33:28.931 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:28.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:33:28.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[107399]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:28.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[107399]: osdmap e551: 8 total, 8 up, 6 in 2024-09-14T04:33:28.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[112165]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:28.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:28.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:28 smithi001 ceph-mon[112165]: osdmap e551: 8 total, 8 up, 6 in 2024-09-14T04:33:29.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:29.060 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:28 smithi012 ceph-mon[101180]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:28 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:28 smithi012 ceph-mon[101180]: osdmap e551: 8 total, 8 up, 6 in 2024-09-14T04:33:29.163 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:29.187 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:29.268 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:33:29.270 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:33:29.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:29.316 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:29.418 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:33:29.445 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:33:29.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:29.574 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:29.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:33:29.702 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: docker: command not found 2024-09-14T04:33:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi012 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:33:29.759 INFO:tasks.workunit.client.0.smithi001.stdout:[----------] 70 tests from TestClsRbd (5069 ms total) 2024-09-14T04:33:29.759 INFO:tasks.workunit.client.0.smithi001.stdout: 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout:[----------] Global test environment tear-down 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout:[==========] 70 tests from 1 test suite ran. (7136 ms total) 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout:[ PASSED ] 69 tests. 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout:[ FAILED ] 1 test, listed below: 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout: 2024-09-14T04:33:29.760 INFO:tasks.workunit.client.0.smithi001.stdout: 1 FAILED TEST 2024-09-14T04:33:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.791 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-14T04:33:29.791 DEBUG:teuthology.orchestra.run.smithi001:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T04:33:29.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:33:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.860 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:29.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:33:29.961 DEBUG:teuthology.orchestra.run.smithi001:> sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:33:29.983 INFO:teuthology.orchestra.run.smithi001.stderr:sudo: docker: command not found 2024-09-14T04:33:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:33:29.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi001 with status 1: 'sudo docker kill -s 1 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:33:30.076 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[107399]: osdmap e552: 8 total, 8 up, 6 in 2024-09-14T04:33:30.076 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[107399]: from='client.36588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:30.076 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[107399]: pgmap v1750: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.7 MiB/s rd, 857 KiB/s wr, 507 op/s 2024-09-14T04:33:30.077 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[112165]: osdmap e552: 8 total, 8 up, 6 in 2024-09-14T04:33:30.077 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[112165]: from='client.36588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:30.077 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:29 smithi001 ceph-mon[112165]: pgmap v1750: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.7 MiB/s rd, 857 KiB/s wr, 507 op/s 2024-09-14T04:33:30.078 INFO:tasks.thrashosds:joining thrashosds 2024-09-14T04:33:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:29 smithi012 ceph-mon[101180]: osdmap e552: 8 total, 8 up, 6 in 2024-09-14T04:33:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:29 smithi012 ceph-mon[101180]: from='client.36588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:29 smithi012 ceph-mon[101180]: pgmap v1750: 125 pgs: 125 active+clean; 396 KiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 1.7 MiB/s rd, 857 KiB/s wr, 507 op/s 2024-09-14T04:33:30.721 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:33:31.086 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:30 smithi001 ceph-mon[112165]: osdmap e553: 8 total, 8 up, 6 in 2024-09-14T04:33:31.086 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:30 smithi001 ceph-mon[107399]: osdmap e553: 8 total, 8 up, 6 in 2024-09-14T04:33:31.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:30 smithi012 ceph-mon[101180]: osdmap e553: 8 total, 8 up, 6 in 2024-09-14T04:33:31.502 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1749,"stamp":"2024-09-14T04:33:28.749030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":405187,"num_objects":59,"num_object_clones":0,"num_object_copies":177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59,"num_whiteouts":0,"num_read":2029,"num_read_kb":10173,"num_write":1113,"num_write_kb":5407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":2465972,"num_keys_recovered":20,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4002,"ondisk_log_size":4002,"up":375,"acting":375,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":326,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7286564,"kb_used_data":49272,"kb_used_omap":25,"kb_used_meta":7237158,"kb_avail":555184348,"statfs":{"total":575970213888,"available":568508772352,"internally_reserved":0,"allocated":50454528,"data_stored":28852325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26477,"internal_metadata":7410849939},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":374,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":1984,"num_read_kb":10073,"num_write":1024,"num_write_kb":5082,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":26,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.095066"},"pg_stats":[{"pgid":"15.1f","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740029+0000","last_change":"2024-09-14T04:33:23.750064+0000","last_active":"2024-09-14T04:33:24.740029+0000","last_peered":"2024-09-14T04:33:24.740029+0000","last_clean":"2024-09-14T04:33:24.740029+0000","last_became_active":"2024-09-14T04:33:23.749713+0000","last_became_peered":"2024-09-14T04:33:23.749713+0000","last_unstale":"2024-09-14T04:33:24.740029+0000","last_undegraded":"2024-09-14T04:33:24.740029+0000","last_fullsized":"2024-09-14T04:33:24.740029+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:55:04.772167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"86'149","reported_seq":906,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740671+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:33:24.740671+0000","last_peered":"2024-09-14T04:33:24.740671+0000","last_clean":"2024-09-14T04:33:24.740671+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:33:24.740671+0000","last_undegraded":"2024-09-14T04:33:24.740671+0000","last_fullsized":"2024-09-14T04:33:24.740671+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.734357+0000","last_change":"2024-09-14T04:30:04.281338+0000","last_active":"2024-09-14T04:33:23.734357+0000","last_peered":"2024-09-14T04:33:23.734357+0000","last_clean":"2024-09-14T04:33:23.734357+0000","last_became_active":"2024-09-14T04:30:04.280363+0000","last_became_peered":"2024-09-14T04:30:04.280363+0000","last_unstale":"2024-09-14T04:33:23.734357+0000","last_undegraded":"2024-09-14T04:33:23.734357+0000","last_fullsized":"2024-09-14T04:33:23.734357+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:50:56.470339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":142,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733087+0000","last_change":"2024-09-14T04:29:37.998593+0000","last_active":"2024-09-14T04:33:23.733087+0000","last_peered":"2024-09-14T04:33:23.733087+0000","last_clean":"2024-09-14T04:33:23.733087+0000","last_became_active":"2024-09-14T04:29:37.991857+0000","last_became_peered":"2024-09-14T04:29:37.991857+0000","last_unstale":"2024-09-14T04:33:23.733087+0000","last_undegraded":"2024-09-14T04:33:23.733087+0000","last_fullsized":"2024-09-14T04:33:23.733087+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:44:08.110065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1e","version":"549'7","reported_seq":19,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739017+0000","last_change":"2024-09-14T04:33:23.745217+0000","last_active":"2024-09-14T04:33:24.739017+0000","last_peered":"2024-09-14T04:33:24.739017+0000","last_clean":"2024-09-14T04:33:24.739017+0000","last_became_active":"2024-09-14T04:33:23.744934+0000","last_became_peered":"2024-09-14T04:33:23.744934+0000","last_unstale":"2024-09-14T04:33:24.739017+0000","last_undegraded":"2024-09-14T04:33:24.739017+0000","last_fullsized":"2024-09-14T04:33:24.739017+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:40:47.301115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":869,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730616+0000","last_change":"2024-09-14T04:32:55.316666+0000","last_active":"2024-09-14T04:33:23.730616+0000","last_peered":"2024-09-14T04:33:23.730616+0000","last_clean":"2024-09-14T04:33:23.730616+0000","last_became_active":"2024-09-14T04:32:55.316301+0000","last_became_peered":"2024-09-14T04:32:55.316301+0000","last_unstale":"2024-09-14T04:33:23.730616+0000","last_undegraded":"2024-09-14T04:33:23.730616+0000","last_fullsized":"2024-09-14T04:33:23.730616+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:15:16.745472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.734902+0000","last_change":"2024-09-14T04:29:18.666490+0000","last_active":"2024-09-14T04:33:23.734902+0000","last_peered":"2024-09-14T04:33:23.734902+0000","last_clean":"2024-09-14T04:33:23.734902+0000","last_became_active":"2024-09-14T04:29:18.666113+0000","last_became_peered":"2024-09-14T04:29:18.666113+0000","last_unstale":"2024-09-14T04:33:23.734902+0000","last_undegraded":"2024-09-14T04:33:23.734902+0000","last_fullsized":"2024-09-14T04:33:23.734902+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:30.604568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":729,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741038+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:33:24.741038+0000","last_peered":"2024-09-14T04:33:24.741038+0000","last_clean":"2024-09-14T04:33:24.741038+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:33:24.741038+0000","last_undegraded":"2024-09-14T04:33:24.741038+0000","last_fullsized":"2024-09-14T04:33:24.741038+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1d","version":"549'2","reported_seq":13,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.325166+0000","last_change":"2024-09-14T04:33:23.751792+0000","last_active":"2024-09-14T04:33:25.325166+0000","last_peered":"2024-09-14T04:33:25.325166+0000","last_clean":"2024-09-14T04:33:25.325166+0000","last_became_active":"2024-09-14T04:33:23.749444+0000","last_became_peered":"2024-09-14T04:33:23.749444+0000","last_unstale":"2024-09-14T04:33:25.325166+0000","last_undegraded":"2024-09-14T04:33:25.325166+0000","last_fullsized":"2024-09-14T04:33:25.325166+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:50.544235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":1005,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.743749+0000","last_change":"2024-09-14T04:30:07.304986+0000","last_active":"2024-09-14T04:33:24.743749+0000","last_peered":"2024-09-14T04:33:24.743749+0000","last_clean":"2024-09-14T04:33:24.743749+0000","last_became_active":"2024-09-14T04:30:07.304452+0000","last_became_peered":"2024-09-14T04:30:07.304452+0000","last_unstale":"2024-09-14T04:33:24.743749+0000","last_undegraded":"2024-09-14T04:33:24.743749+0000","last_fullsized":"2024-09-14T04:33:24.743749+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:59.521970+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":715,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742231+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:33:24.742231+0000","last_peered":"2024-09-14T04:33:24.742231+0000","last_clean":"2024-09-14T04:33:24.742231+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:33:24.742231+0000","last_undegraded":"2024-09-14T04:33:24.742231+0000","last_fullsized":"2024-09-14T04:33:24.742231+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":130,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730960+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:33:23.730960+0000","last_peered":"2024-09-14T04:33:23.730960+0000","last_clean":"2024-09-14T04:33:23.730960+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:33:23.730960+0000","last_undegraded":"2024-09-14T04:33:23.730960+0000","last_fullsized":"2024-09-14T04:33:23.730960+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324773+0000","last_change":"2024-09-14T04:33:23.751896+0000","last_active":"2024-09-14T04:33:25.324773+0000","last_peered":"2024-09-14T04:33:25.324773+0000","last_clean":"2024-09-14T04:33:25.324773+0000","last_became_active":"2024-09-14T04:33:23.751453+0000","last_became_peered":"2024-09-14T04:33:23.751453+0000","last_unstale":"2024-09-14T04:33:25.324773+0000","last_undegraded":"2024-09-14T04:33:25.324773+0000","last_fullsized":"2024-09-14T04:33:25.324773+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:34:42.928750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":708,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732266+0000","last_change":"2024-09-14T04:30:07.297746+0000","last_active":"2024-09-14T04:33:23.732266+0000","last_peered":"2024-09-14T04:33:23.732266+0000","last_clean":"2024-09-14T04:33:23.732266+0000","last_became_active":"2024-09-14T04:30:07.297186+0000","last_became_peered":"2024-09-14T04:30:07.297186+0000","last_unstale":"2024-09-14T04:33:23.732266+0000","last_undegraded":"2024-09-14T04:33:23.732266+0000","last_fullsized":"2024-09-14T04:33:23.732266+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:59:40.613090+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":301,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.744549+0000","last_change":"2024-09-14T04:30:04.286496+0000","last_active":"2024-09-14T04:33:24.744549+0000","last_peered":"2024-09-14T04:33:24.744549+0000","last_clean":"2024-09-14T04:33:24.744549+0000","last_became_active":"2024-09-14T04:30:04.282753+0000","last_became_peered":"2024-09-14T04:30:04.282753+0000","last_unstale":"2024-09-14T04:33:24.744549+0000","last_undegraded":"2024-09-14T04:33:24.744549+0000","last_fullsized":"2024-09-14T04:33:24.744549+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:39.799809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":112,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732070+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:33:23.732070+0000","last_peered":"2024-09-14T04:33:23.732070+0000","last_clean":"2024-09-14T04:33:23.732070+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:33:23.732070+0000","last_undegraded":"2024-09-14T04:33:23.732070+0000","last_fullsized":"2024-09-14T04:33:23.732070+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.325113+0000","last_change":"2024-09-14T04:33:23.754222+0000","last_active":"2024-09-14T04:33:25.325113+0000","last_peered":"2024-09-14T04:33:25.325113+0000","last_clean":"2024-09-14T04:33:25.325113+0000","last_became_active":"2024-09-14T04:33:23.753778+0000","last_became_peered":"2024-09-14T04:33:23.753778+0000","last_unstale":"2024-09-14T04:33:25.325113+0000","last_undegraded":"2024-09-14T04:33:25.325113+0000","last_fullsized":"2024-09-14T04:33:25.325113+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:52.112221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":837,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.735061+0000","last_change":"2024-09-14T04:30:07.303908+0000","last_active":"2024-09-14T04:33:23.735061+0000","last_peered":"2024-09-14T04:33:23.735061+0000","last_clean":"2024-09-14T04:33:23.735061+0000","last_became_active":"2024-09-14T04:30:07.302591+0000","last_became_peered":"2024-09-14T04:30:07.302591+0000","last_unstale":"2024-09-14T04:33:23.735061+0000","last_undegraded":"2024-09-14T04:33:23.735061+0000","last_fullsized":"2024-09-14T04:33:23.735061+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:51.333058+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":125,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731977+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:33:23.731977+0000","last_peered":"2024-09-14T04:33:23.731977+0000","last_clean":"2024-09-14T04:33:23.731977+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:33:23.731977+0000","last_undegraded":"2024-09-14T04:33:23.731977+0000","last_fullsized":"2024-09-14T04:33:23.731977+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730173+0000","last_change":"2024-09-14T04:32:52.814891+0000","last_active":"2024-09-14T04:33:23.730173+0000","last_peered":"2024-09-14T04:33:23.730173+0000","last_clean":"2024-09-14T04:33:23.730173+0000","last_became_active":"2024-09-14T04:32:52.814547+0000","last_became_peered":"2024-09-14T04:32:52.814547+0000","last_unstale":"2024-09-14T04:33:23.730173+0000","last_undegraded":"2024-09-14T04:33:23.730173+0000","last_fullsized":"2024-09-14T04:33:23.730173+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:43:14.342218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.2","version":"549'1","reported_seq":13,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324561+0000","last_change":"2024-09-14T04:33:23.753398+0000","last_active":"2024-09-14T04:33:25.324561+0000","last_peered":"2024-09-14T04:33:25.324561+0000","last_clean":"2024-09-14T04:33:25.324561+0000","last_became_active":"2024-09-14T04:33:23.752623+0000","last_became_peered":"2024-09-14T04:33:23.752623+0000","last_unstale":"2024-09-14T04:33:25.324561+0000","last_undegraded":"2024-09-14T04:33:25.324561+0000","last_fullsized":"2024-09-14T04:33:25.324561+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:39.477882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1263,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.744104+0000","last_change":"2024-09-14T04:28:22.698300+0000","last_active":"2024-09-14T04:33:24.744104+0000","last_peered":"2024-09-14T04:33:24.744104+0000","last_clean":"2024-09-14T04:33:24.744104+0000","last_became_active":"2024-09-14T04:28:22.697894+0000","last_became_peered":"2024-09-14T04:28:22.697894+0000","last_unstale":"2024-09-14T04:33:24.744104+0000","last_undegraded":"2024-09-14T04:33:24.744104+0000","last_fullsized":"2024-09-14T04:33:24.744104+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:19.688955+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":138,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733158+0000","last_change":"2024-09-14T04:30:04.302042+0000","last_active":"2024-09-14T04:33:23.733158+0000","last_peered":"2024-09-14T04:33:23.733158+0000","last_clean":"2024-09-14T04:33:23.733158+0000","last_became_active":"2024-09-14T04:30:04.281337+0000","last_became_peered":"2024-09-14T04:30:04.281337+0000","last_unstale":"2024-09-14T04:33:23.733158+0000","last_undegraded":"2024-09-14T04:33:23.733158+0000","last_fullsized":"2024-09-14T04:33:23.733158+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:39.377188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":597,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747550+0000","last_change":"2024-09-14T04:32:52.818598+0000","last_active":"2024-09-14T04:33:23.747550+0000","last_peered":"2024-09-14T04:33:23.747550+0000","last_clean":"2024-09-14T04:33:23.747550+0000","last_became_active":"2024-09-14T04:32:52.818281+0000","last_became_peered":"2024-09-14T04:32:52.818281+0000","last_unstale":"2024-09-14T04:33:23.747550+0000","last_undegraded":"2024-09-14T04:33:23.747550+0000","last_fullsized":"2024-09-14T04:33:23.747550+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:39.775021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.1","version":"549'15","reported_seq":26,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739154+0000","last_change":"2024-09-14T04:33:23.740524+0000","last_active":"2024-09-14T04:33:24.739154+0000","last_peered":"2024-09-14T04:33:24.739154+0000","last_clean":"2024-09-14T04:33:24.739154+0000","last_became_active":"2024-09-14T04:33:23.739969+0000","last_became_peered":"2024-09-14T04:33:23.739969+0000","last_unstale":"2024-09-14T04:33:24.739154+0000","last_undegraded":"2024-09-14T04:33:24.739154+0000","last_fullsized":"2024-09-14T04:33:24.739154+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:54:36.002275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":501,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733886+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:33:23.733886+0000","last_peered":"2024-09-14T04:33:23.733886+0000","last_clean":"2024-09-14T04:33:23.733886+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:33:23.733886+0000","last_undegraded":"2024-09-14T04:33:23.733886+0000","last_fullsized":"2024-09-14T04:33:23.733886+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":138,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.735170+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:33:23.735170+0000","last_peered":"2024-09-14T04:33:23.735170+0000","last_clean":"2024-09-14T04:33:23.735170+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:33:23.735170+0000","last_undegraded":"2024-09-14T04:33:23.735170+0000","last_fullsized":"2024-09-14T04:33:23.735170+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732379+0000","last_change":"2024-09-14T04:32:52.810817+0000","last_active":"2024-09-14T04:33:23.732379+0000","last_peered":"2024-09-14T04:33:23.732379+0000","last_clean":"2024-09-14T04:33:23.732379+0000","last_became_active":"2024-09-14T04:32:52.809483+0000","last_became_peered":"2024-09-14T04:32:52.809483+0000","last_unstale":"2024-09-14T04:33:23.732379+0000","last_undegraded":"2024-09-14T04:33:23.732379+0000","last_fullsized":"2024-09-14T04:33:23.732379+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:00.995883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"549'3","reported_seq":16,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324554+0000","last_change":"2024-09-14T04:33:23.749572+0000","last_active":"2024-09-14T04:33:25.324554+0000","last_peered":"2024-09-14T04:33:25.324554+0000","last_clean":"2024-09-14T04:33:25.324554+0000","last_became_active":"2024-09-14T04:33:23.749069+0000","last_became_peered":"2024-09-14T04:33:23.749069+0000","last_unstale":"2024-09-14T04:33:25.324554+0000","last_undegraded":"2024-09-14T04:33:25.324554+0000","last_fullsized":"2024-09-14T04:33:25.324554+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:10:26.768303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":3,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":558,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.746646+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:33:23.746646+0000","last_peered":"2024-09-14T04:33:23.746646+0000","last_clean":"2024-09-14T04:33:23.746646+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:33:23.746646+0000","last_undegraded":"2024-09-14T04:33:23.746646+0000","last_fullsized":"2024-09-14T04:33:23.746646+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":547,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.743642+0000","last_change":"2024-09-14T04:29:18.673642+0000","last_active":"2024-09-14T04:33:24.743642+0000","last_peered":"2024-09-14T04:33:24.743642+0000","last_clean":"2024-09-14T04:33:24.743642+0000","last_became_active":"2024-09-14T04:29:18.673365+0000","last_became_peered":"2024-09-14T04:29:18.673365+0000","last_unstale":"2024-09-14T04:33:24.743642+0000","last_undegraded":"2024-09-14T04:33:24.743642+0000","last_fullsized":"2024-09-14T04:33:24.743642+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:32.707692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":709,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742234+0000","last_change":"2024-09-14T04:30:04.292857+0000","last_active":"2024-09-14T04:33:24.742234+0000","last_peered":"2024-09-14T04:33:24.742234+0000","last_clean":"2024-09-14T04:33:24.742234+0000","last_became_active":"2024-09-14T04:30:04.292434+0000","last_became_peered":"2024-09-14T04:30:04.292434+0000","last_unstale":"2024-09-14T04:33:24.742234+0000","last_undegraded":"2024-09-14T04:33:24.742234+0000","last_fullsized":"2024-09-14T04:33:24.742234+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:28.993975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.7","version":"549'23","reported_seq":58,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742376+0000","last_change":"2024-09-14T04:33:23.746581+0000","last_active":"2024-09-14T04:33:24.742376+0000","last_peered":"2024-09-14T04:33:24.742376+0000","last_clean":"2024-09-14T04:33:24.742376+0000","last_became_active":"2024-09-14T04:33:23.746229+0000","last_became_peered":"2024-09-14T04:33:23.746229+0000","last_unstale":"2024-09-14T04:33:24.742376+0000","last_undegraded":"2024-09-14T04:33:24.742376+0000","last_fullsized":"2024-09-14T04:33:24.742376+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:10:00.285373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":904,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731151+0000","last_change":"2024-09-14T04:29:40.287295+0000","last_active":"2024-09-14T04:33:23.731151+0000","last_peered":"2024-09-14T04:33:23.731151+0000","last_clean":"2024-09-14T04:33:23.731151+0000","last_became_active":"2024-09-14T04:29:40.286891+0000","last_became_peered":"2024-09-14T04:29:40.286891+0000","last_unstale":"2024-09-14T04:33:23.731151+0000","last_undegraded":"2024-09-14T04:33:23.731151+0000","last_fullsized":"2024-09-14T04:33:23.731151+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:59.160582+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":706,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741628+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:33:24.741628+0000","last_peered":"2024-09-14T04:33:24.741628+0000","last_clean":"2024-09-14T04:33:24.741628+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:33:24.741628+0000","last_undegraded":"2024-09-14T04:33:24.741628+0000","last_fullsized":"2024-09-14T04:33:24.741628+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":590,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747270+0000","last_change":"2024-09-14T04:32:52.819885+0000","last_active":"2024-09-14T04:33:23.747270+0000","last_peered":"2024-09-14T04:33:23.747270+0000","last_clean":"2024-09-14T04:33:23.747270+0000","last_became_active":"2024-09-14T04:32:52.819034+0000","last_became_peered":"2024-09-14T04:32:52.819034+0000","last_unstale":"2024-09-14T04:33:23.747270+0000","last_undegraded":"2024-09-14T04:33:23.747270+0000","last_fullsized":"2024-09-14T04:33:23.747270+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:52.327969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.6","version":"550'581","reported_seq":644,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.901806+0000","last_change":"2024-09-14T04:33:23.742530+0000","last_active":"2024-09-14T04:33:24.901806+0000","last_peered":"2024-09-14T04:33:24.901806+0000","last_clean":"2024-09-14T04:33:24.901806+0000","last_became_active":"2024-09-14T04:33:23.742198+0000","last_became_peered":"2024-09-14T04:33:23.742198+0000","last_unstale":"2024-09-14T04:33:24.901806+0000","last_undegraded":"2024-09-14T04:33:24.901806+0000","last_fullsized":"2024-09-14T04:33:24.901806+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":581,"log_dups_size":0,"ondisk_log_size":581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:39:34.438922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1286,"num_read_kb":1253,"num_write":838,"num_write_kb":801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":379,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731448+0000","last_change":"2024-09-14T04:32:59.331716+0000","last_active":"2024-09-14T04:33:23.731448+0000","last_peered":"2024-09-14T04:33:23.731448+0000","last_clean":"2024-09-14T04:33:23.731448+0000","last_became_active":"2024-09-14T04:32:59.331294+0000","last_became_peered":"2024-09-14T04:32:59.331294+0000","last_unstale":"2024-09-14T04:33:23.731448+0000","last_undegraded":"2024-09-14T04:33:23.731448+0000","last_fullsized":"2024-09-14T04:33:23.731448+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732493+0000","last_change":"2024-09-14T04:30:04.285245+0000","last_active":"2024-09-14T04:33:23.732493+0000","last_peered":"2024-09-14T04:33:23.732493+0000","last_clean":"2024-09-14T04:33:23.732493+0000","last_became_active":"2024-09-14T04:30:04.284576+0000","last_became_peered":"2024-09-14T04:30:04.284576+0000","last_unstale":"2024-09-14T04:33:23.732493+0000","last_undegraded":"2024-09-14T04:33:23.732493+0000","last_fullsized":"2024-09-14T04:33:23.732493+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:15.875331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.748269+0000","last_change":"2024-09-14T04:30:04.295569+0000","last_active":"2024-09-14T04:33:23.748269+0000","last_peered":"2024-09-14T04:33:23.748269+0000","last_clean":"2024-09-14T04:33:23.748269+0000","last_became_active":"2024-09-14T04:30:04.294571+0000","last_became_peered":"2024-09-14T04:30:04.294571+0000","last_unstale":"2024-09-14T04:33:23.748269+0000","last_undegraded":"2024-09-14T04:33:23.748269+0000","last_fullsized":"2024-09-14T04:33:23.748269+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:40.129021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739237+0000","last_change":"2024-09-14T04:33:23.746136+0000","last_active":"2024-09-14T04:33:24.739237+0000","last_peered":"2024-09-14T04:33:24.739237+0000","last_clean":"2024-09-14T04:33:24.739237+0000","last_became_active":"2024-09-14T04:33:23.745825+0000","last_became_peered":"2024-09-14T04:33:23.745825+0000","last_unstale":"2024-09-14T04:33:24.739237+0000","last_undegraded":"2024-09-14T04:33:24.739237+0000","last_fullsized":"2024-09-14T04:33:24.739237+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:52.705745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":951,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741438+0000","last_change":"2024-09-14T04:32:55.317365+0000","last_active":"2024-09-14T04:33:24.741438+0000","last_peered":"2024-09-14T04:33:24.741438+0000","last_clean":"2024-09-14T04:33:24.741438+0000","last_became_active":"2024-09-14T04:32:55.316994+0000","last_became_peered":"2024-09-14T04:32:55.316994+0000","last_unstale":"2024-09-14T04:33:24.741438+0000","last_undegraded":"2024-09-14T04:33:24.741438+0000","last_fullsized":"2024-09-14T04:33:24.741438+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:17.732488+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":189,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747294+0000","last_change":"2024-09-14T04:30:04.291481+0000","last_active":"2024-09-14T04:33:23.747294+0000","last_peered":"2024-09-14T04:33:23.747294+0000","last_clean":"2024-09-14T04:33:23.747294+0000","last_became_active":"2024-09-14T04:30:04.291034+0000","last_became_peered":"2024-09-14T04:30:04.291034+0000","last_unstale":"2024-09-14T04:33:23.747294+0000","last_undegraded":"2024-09-14T04:33:23.747294+0000","last_fullsized":"2024-09-14T04:33:23.747294+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:09.399818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":587,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747371+0000","last_change":"2024-09-14T04:32:52.817581+0000","last_active":"2024-09-14T04:33:23.747371+0000","last_peered":"2024-09-14T04:33:23.747371+0000","last_clean":"2024-09-14T04:33:23.747371+0000","last_became_active":"2024-09-14T04:32:52.817276+0000","last_became_peered":"2024-09-14T04:32:52.817276+0000","last_unstale":"2024-09-14T04:33:23.747371+0000","last_undegraded":"2024-09-14T04:33:23.747371+0000","last_fullsized":"2024-09-14T04:33:23.747371+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:27.119610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739900+0000","last_change":"2024-09-14T04:33:23.740746+0000","last_active":"2024-09-14T04:33:24.739900+0000","last_peered":"2024-09-14T04:33:24.739900+0000","last_clean":"2024-09-14T04:33:24.739900+0000","last_became_active":"2024-09-14T04:33:23.740464+0000","last_became_peered":"2024-09-14T04:33:23.740464+0000","last_unstale":"2024-09-14T04:33:24.739900+0000","last_undegraded":"2024-09-14T04:33:24.739900+0000","last_fullsized":"2024-09-14T04:33:24.739900+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:52.614938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":739,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732267+0000","last_change":"2024-09-14T04:29:40.277117+0000","last_active":"2024-09-14T04:33:23.732267+0000","last_peered":"2024-09-14T04:33:23.732267+0000","last_clean":"2024-09-14T04:33:23.732267+0000","last_became_active":"2024-09-14T04:29:40.276702+0000","last_became_peered":"2024-09-14T04:29:40.276702+0000","last_unstale":"2024-09-14T04:33:23.732267+0000","last_undegraded":"2024-09-14T04:33:23.732267+0000","last_fullsized":"2024-09-14T04:33:23.732267+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:19.375908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:19.375908+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:20.933166+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":301,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.743285+0000","last_change":"2024-09-14T04:30:04.298649+0000","last_active":"2024-09-14T04:33:24.743285+0000","last_peered":"2024-09-14T04:33:24.743285+0000","last_clean":"2024-09-14T04:33:24.743285+0000","last_became_active":"2024-09-14T04:30:04.295253+0000","last_became_peered":"2024-09-14T04:30:04.295253+0000","last_unstale":"2024-09-14T04:33:24.743285+0000","last_undegraded":"2024-09-14T04:33:24.743285+0000","last_fullsized":"2024-09-14T04:33:24.743285+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:23.430095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747348+0000","last_change":"2024-09-14T04:30:04.296998+0000","last_active":"2024-09-14T04:33:23.747348+0000","last_peered":"2024-09-14T04:33:23.747348+0000","last_clean":"2024-09-14T04:33:23.747348+0000","last_became_active":"2024-09-14T04:30:04.296685+0000","last_became_peered":"2024-09-14T04:30:04.296685+0000","last_unstale":"2024-09-14T04:33:23.747348+0000","last_undegraded":"2024-09-14T04:33:23.747348+0000","last_fullsized":"2024-09-14T04:33:23.747348+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:31.564351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.e","version":"549'9","reported_seq":48,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.325176+0000","last_change":"2024-09-14T04:33:23.753431+0000","last_active":"2024-09-14T04:33:25.325176+0000","last_peered":"2024-09-14T04:33:25.325176+0000","last_clean":"2024-09-14T04:33:25.325176+0000","last_became_active":"2024-09-14T04:33:23.751282+0000","last_became_peered":"2024-09-14T04:33:23.751282+0000","last_unstale":"2024-09-14T04:33:25.325176+0000","last_undegraded":"2024-09-14T04:33:25.325176+0000","last_fullsized":"2024-09-14T04:33:25.325176+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:35.237377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":52,"num_write":7,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":960,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730531+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:33:23.730531+0000","last_peered":"2024-09-14T04:33:23.730531+0000","last_clean":"2024-09-14T04:33:23.730531+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:33:23.730531+0000","last_undegraded":"2024-09-14T04:33:23.730531+0000","last_fullsized":"2024-09-14T04:33:23.730531+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":451,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732994+0000","last_change":"2024-09-14T04:29:38.001028+0000","last_active":"2024-09-14T04:33:23.732994+0000","last_peered":"2024-09-14T04:33:23.732994+0000","last_clean":"2024-09-14T04:33:23.732994+0000","last_became_active":"2024-09-14T04:29:37.987180+0000","last_became_peered":"2024-09-14T04:29:37.987180+0000","last_unstale":"2024-09-14T04:33:23.732994+0000","last_undegraded":"2024-09-14T04:33:23.732994+0000","last_fullsized":"2024-09-14T04:33:23.732994+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:29.081173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":151,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732383+0000","last_change":"2024-09-14T04:29:37.992219+0000","last_active":"2024-09-14T04:33:23.732383+0000","last_peered":"2024-09-14T04:33:23.732383+0000","last_clean":"2024-09-14T04:33:23.732383+0000","last_became_active":"2024-09-14T04:29:37.991324+0000","last_became_peered":"2024-09-14T04:29:37.991324+0000","last_unstale":"2024-09-14T04:33:23.732383+0000","last_undegraded":"2024-09-14T04:33:23.732383+0000","last_fullsized":"2024-09-14T04:33:23.732383+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:08.893407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732643+0000","last_change":"2024-09-14T04:32:52.817519+0000","last_active":"2024-09-14T04:33:23.732643+0000","last_peered":"2024-09-14T04:33:23.732643+0000","last_clean":"2024-09-14T04:33:23.732643+0000","last_became_active":"2024-09-14T04:32:52.817207+0000","last_became_peered":"2024-09-14T04:32:52.817207+0000","last_unstale":"2024-09-14T04:33:23.732643+0000","last_undegraded":"2024-09-14T04:33:23.732643+0000","last_fullsized":"2024-09-14T04:33:23.732643+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:43.967076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"549'30","reported_seq":66,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.738304+0000","last_change":"2024-09-14T04:33:23.744285+0000","last_active":"2024-09-14T04:33:24.738304+0000","last_peered":"2024-09-14T04:33:24.738304+0000","last_clean":"2024-09-14T04:33:24.738304+0000","last_became_active":"2024-09-14T04:33:23.744002+0000","last_became_peered":"2024-09-14T04:33:23.744002+0000","last_unstale":"2024-09-14T04:33:24.738304+0000","last_undegraded":"2024-09-14T04:33:24.738304+0000","last_fullsized":"2024-09-14T04:33:24.738304+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:17.846360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":139,"num_read_kb":104,"num_write":29,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":387,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731803+0000","last_change":"2024-09-14T04:32:55.318369+0000","last_active":"2024-09-14T04:33:23.731803+0000","last_peered":"2024-09-14T04:33:23.731803+0000","last_clean":"2024-09-14T04:33:23.731803+0000","last_became_active":"2024-09-14T04:32:55.317929+0000","last_became_peered":"2024-09-14T04:32:55.317929+0000","last_unstale":"2024-09-14T04:33:23.731803+0000","last_undegraded":"2024-09-14T04:33:23.731803+0000","last_fullsized":"2024-09-14T04:33:23.731803+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730346+0000","last_change":"2024-09-14T04:30:04.283624+0000","last_active":"2024-09-14T04:33:23.730346+0000","last_peered":"2024-09-14T04:33:23.730346+0000","last_clean":"2024-09-14T04:33:23.730346+0000","last_became_active":"2024-09-14T04:30:04.283298+0000","last_became_peered":"2024-09-14T04:30:04.283298+0000","last_unstale":"2024-09-14T04:33:23.730346+0000","last_undegraded":"2024-09-14T04:33:23.730346+0000","last_fullsized":"2024-09-14T04:33:23.730346+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:32.022169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":952,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740464+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:33:24.740464+0000","last_peered":"2024-09-14T04:33:24.740464+0000","last_clean":"2024-09-14T04:33:24.740464+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:33:24.740464+0000","last_undegraded":"2024-09-14T04:33:24.740464+0000","last_fullsized":"2024-09-14T04:33:24.740464+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":727,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742586+0000","last_change":"2024-09-14T04:30:04.285113+0000","last_active":"2024-09-14T04:33:24.742586+0000","last_peered":"2024-09-14T04:33:24.742586+0000","last_clean":"2024-09-14T04:33:24.742586+0000","last_became_active":"2024-09-14T04:30:04.283783+0000","last_became_peered":"2024-09-14T04:30:04.283783+0000","last_unstale":"2024-09-14T04:33:24.742586+0000","last_undegraded":"2024-09-14T04:33:24.742586+0000","last_fullsized":"2024-09-14T04:33:24.742586+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:13.747229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.c","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739819+0000","last_change":"2024-09-14T04:33:23.741486+0000","last_active":"2024-09-14T04:33:24.739819+0000","last_peered":"2024-09-14T04:33:24.739819+0000","last_clean":"2024-09-14T04:33:24.739819+0000","last_became_active":"2024-09-14T04:33:23.741189+0000","last_became_peered":"2024-09-14T04:33:23.741189+0000","last_unstale":"2024-09-14T04:33:24.739819+0000","last_undegraded":"2024-09-14T04:33:24.739819+0000","last_fullsized":"2024-09-14T04:33:24.739819+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:49:53.282446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":817,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733989+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:33:23.733989+0000","last_peered":"2024-09-14T04:33:23.733989+0000","last_clean":"2024-09-14T04:33:23.733989+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:33:23.733989+0000","last_undegraded":"2024-09-14T04:33:23.733989+0000","last_fullsized":"2024-09-14T04:33:23.733989+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":520,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740783+0000","last_change":"2024-09-14T04:30:04.285510+0000","last_active":"2024-09-14T04:33:24.740783+0000","last_peered":"2024-09-14T04:33:24.740783+0000","last_clean":"2024-09-14T04:33:24.740783+0000","last_became_active":"2024-09-14T04:30:04.285170+0000","last_became_peered":"2024-09-14T04:30:04.285170+0000","last_unstale":"2024-09-14T04:33:24.740783+0000","last_undegraded":"2024-09-14T04:33:24.740783+0000","last_fullsized":"2024-09-14T04:33:24.740783+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:32.780039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731824+0000","last_change":"2024-09-14T04:32:52.815265+0000","last_active":"2024-09-14T04:33:23.731824+0000","last_peered":"2024-09-14T04:33:23.731824+0000","last_clean":"2024-09-14T04:33:23.731824+0000","last_became_active":"2024-09-14T04:32:52.814764+0000","last_became_peered":"2024-09-14T04:32:52.814764+0000","last_unstale":"2024-09-14T04:33:23.731824+0000","last_undegraded":"2024-09-14T04:33:23.731824+0000","last_fullsized":"2024-09-14T04:33:23.731824+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:59.683211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":479,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741957+0000","last_change":"2024-09-14T04:30:04.287957+0000","last_active":"2024-09-14T04:33:24.741957+0000","last_peered":"2024-09-14T04:33:24.741957+0000","last_clean":"2024-09-14T04:33:24.741957+0000","last_became_active":"2024-09-14T04:30:04.287313+0000","last_became_peered":"2024-09-14T04:30:04.287313+0000","last_unstale":"2024-09-14T04:33:24.741957+0000","last_undegraded":"2024-09-14T04:33:24.741957+0000","last_fullsized":"2024-09-14T04:33:24.741957+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:25.063506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.f","version":"549'10","reported_seq":26,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740126+0000","last_change":"2024-09-14T04:33:23.746197+0000","last_active":"2024-09-14T04:33:24.740126+0000","last_peered":"2024-09-14T04:33:24.740126+0000","last_clean":"2024-09-14T04:33:24.740126+0000","last_became_active":"2024-09-14T04:33:23.745891+0000","last_became_peered":"2024-09-14T04:33:23.745891+0000","last_unstale":"2024-09-14T04:33:24.740126+0000","last_undegraded":"2024-09-14T04:33:24.740126+0000","last_fullsized":"2024-09-14T04:33:24.740126+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:49:53.458523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":18,"num_read_kb":8199,"num_write":8,"num_write_kb":4100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":698,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740980+0000","last_change":"2024-09-14T04:29:40.285341+0000","last_active":"2024-09-14T04:33:24.740980+0000","last_peered":"2024-09-14T04:33:24.740980+0000","last_clean":"2024-09-14T04:33:24.740980+0000","last_became_active":"2024-09-14T04:29:40.284955+0000","last_became_peered":"2024-09-14T04:29:40.284955+0000","last_unstale":"2024-09-14T04:33:24.740980+0000","last_undegraded":"2024-09-14T04:33:24.740980+0000","last_fullsized":"2024-09-14T04:33:24.740980+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:18:51.087521+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":26,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732159+0000","last_change":"2024-09-14T04:32:57.968797+0000","last_active":"2024-09-14T04:33:23.732159+0000","last_peered":"2024-09-14T04:33:23.732159+0000","last_clean":"2024-09-14T04:33:23.732159+0000","last_became_active":"2024-09-14T04:32:52.810067+0000","last_became_peered":"2024-09-14T04:32:52.810067+0000","last_unstale":"2024-09-14T04:33:23.732159+0000","last_undegraded":"2024-09-14T04:33:23.732159+0000","last_fullsized":"2024-09-14T04:33:23.732159+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:19.561540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":581,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.748945+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:33:23.748945+0000","last_peered":"2024-09-14T04:33:23.748945+0000","last_clean":"2024-09-14T04:33:23.748945+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:33:23.748945+0000","last_undegraded":"2024-09-14T04:33:23.748945+0000","last_fullsized":"2024-09-14T04:33:23.748945+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733290+0000","last_change":"2024-09-14T04:32:52.807897+0000","last_active":"2024-09-14T04:33:23.733290+0000","last_peered":"2024-09-14T04:33:23.733290+0000","last_clean":"2024-09-14T04:33:23.733290+0000","last_became_active":"2024-09-14T04:32:52.806944+0000","last_became_peered":"2024-09-14T04:32:52.806944+0000","last_unstale":"2024-09-14T04:33:23.733290+0000","last_undegraded":"2024-09-14T04:33:23.733290+0000","last_fullsized":"2024-09-14T04:33:23.733290+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:23.347741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.9","version":"549'4","reported_seq":18,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739041+0000","last_change":"2024-09-14T04:33:23.740139+0000","last_active":"2024-09-14T04:33:24.739041+0000","last_peered":"2024-09-14T04:33:24.739041+0000","last_clean":"2024-09-14T04:33:24.739041+0000","last_became_active":"2024-09-14T04:33:23.739845+0000","last_became_peered":"2024-09-14T04:33:23.739845+0000","last_unstale":"2024-09-14T04:33:24.739041+0000","last_undegraded":"2024-09-14T04:33:24.739041+0000","last_fullsized":"2024-09-14T04:33:24.739041+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:32.911978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":945,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733132+0000","last_change":"2024-09-14T04:32:55.315948+0000","last_active":"2024-09-14T04:33:23.733132+0000","last_peered":"2024-09-14T04:33:23.733132+0000","last_clean":"2024-09-14T04:33:23.733132+0000","last_became_active":"2024-09-14T04:32:55.315594+0000","last_became_peered":"2024-09-14T04:32:55.315594+0000","last_unstale":"2024-09-14T04:33:23.733132+0000","last_undegraded":"2024-09-14T04:33:23.733132+0000","last_fullsized":"2024-09-14T04:33:23.733132+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:59.189391+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":568,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.744307+0000","last_change":"2024-09-14T04:29:37.992555+0000","last_active":"2024-09-14T04:33:24.744307+0000","last_peered":"2024-09-14T04:33:24.744307+0000","last_clean":"2024-09-14T04:33:24.744307+0000","last_became_active":"2024-09-14T04:29:37.992147+0000","last_became_peered":"2024-09-14T04:29:37.992147+0000","last_unstale":"2024-09-14T04:33:24.744307+0000","last_undegraded":"2024-09-14T04:33:24.744307+0000","last_fullsized":"2024-09-14T04:33:24.744307+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:26:44.313121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":880,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741468+0000","last_change":"2024-09-14T04:30:04.287689+0000","last_active":"2024-09-14T04:33:24.741468+0000","last_peered":"2024-09-14T04:33:24.741468+0000","last_clean":"2024-09-14T04:33:24.741468+0000","last_became_active":"2024-09-14T04:30:04.287280+0000","last_became_peered":"2024-09-14T04:30:04.287280+0000","last_unstale":"2024-09-14T04:33:24.741468+0000","last_undegraded":"2024-09-14T04:33:24.741468+0000","last_fullsized":"2024-09-14T04:33:24.741468+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:19.006536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":188,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747503+0000","last_change":"2024-09-14T04:30:04.294612+0000","last_active":"2024-09-14T04:33:23.747503+0000","last_peered":"2024-09-14T04:33:23.747503+0000","last_clean":"2024-09-14T04:33:23.747503+0000","last_became_active":"2024-09-14T04:30:04.294288+0000","last_became_peered":"2024-09-14T04:30:04.294288+0000","last_unstale":"2024-09-14T04:33:23.747503+0000","last_undegraded":"2024-09-14T04:33:23.747503+0000","last_fullsized":"2024-09-14T04:33:23.747503+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:58.493527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.8","version":"550'14","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:26.946443+0000","last_change":"2024-09-14T04:33:23.739768+0000","last_active":"2024-09-14T04:33:26.946443+0000","last_peered":"2024-09-14T04:33:26.946443+0000","last_clean":"2024-09-14T04:33:26.946443+0000","last_became_active":"2024-09-14T04:33:23.739461+0000","last_became_peered":"2024-09-14T04:33:23.739461+0000","last_unstale":"2024-09-14T04:33:26.946443+0000","last_undegraded":"2024-09-14T04:33:26.946443+0000","last_fullsized":"2024-09-14T04:33:26.946443+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:15.305996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":23,"num_read_kb":23,"num_write":12,"num_write_kb":11,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":875,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732659+0000","last_change":"2024-09-14T04:28:22.699361+0000","last_active":"2024-09-14T04:33:23.732659+0000","last_peered":"2024-09-14T04:33:23.732659+0000","last_clean":"2024-09-14T04:33:23.732659+0000","last_became_active":"2024-09-14T04:28:22.699020+0000","last_became_peered":"2024-09-14T04:28:22.699020+0000","last_unstale":"2024-09-14T04:33:23.732659+0000","last_undegraded":"2024-09-14T04:33:23.732659+0000","last_fullsized":"2024-09-14T04:33:23.732659+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:07.313428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":943,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741295+0000","last_change":"2024-09-14T04:30:04.279568+0000","last_active":"2024-09-14T04:33:24.741295+0000","last_peered":"2024-09-14T04:33:24.741295+0000","last_clean":"2024-09-14T04:33:24.741295+0000","last_became_active":"2024-09-14T04:30:04.279227+0000","last_became_peered":"2024-09-14T04:30:04.279227+0000","last_unstale":"2024-09-14T04:33:24.741295+0000","last_undegraded":"2024-09-14T04:33:24.741295+0000","last_fullsized":"2024-09-14T04:33:24.741295+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:46.120838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.734996+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:33:23.734996+0000","last_peered":"2024-09-14T04:33:23.734996+0000","last_clean":"2024-09-14T04:33:23.734996+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:33:23.734996+0000","last_undegraded":"2024-09-14T04:33:23.734996+0000","last_fullsized":"2024-09-14T04:33:23.734996+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":725,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742123+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:33:24.742123+0000","last_peered":"2024-09-14T04:33:24.742123+0000","last_clean":"2024-09-14T04:33:24.742123+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:33:24.742123+0000","last_undegraded":"2024-09-14T04:33:24.742123+0000","last_fullsized":"2024-09-14T04:33:24.742123+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.b","version":"549'37","reported_seq":76,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741347+0000","last_change":"2024-09-14T04:33:23.751487+0000","last_active":"2024-09-14T04:33:24.741347+0000","last_peered":"2024-09-14T04:33:24.741347+0000","last_clean":"2024-09-14T04:33:24.741347+0000","last_became_active":"2024-09-14T04:33:23.751226+0000","last_became_peered":"2024-09-14T04:33:23.751226+0000","last_unstale":"2024-09-14T04:33:24.741347+0000","last_undegraded":"2024-09-14T04:33:24.741347+0000","last_fullsized":"2024-09-14T04:33:24.741347+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:59:44.865442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":215,"num_read_kb":174,"num_write":44,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":892,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747885+0000","last_change":"2024-09-14T04:32:55.317272+0000","last_active":"2024-09-14T04:33:23.747885+0000","last_peered":"2024-09-14T04:33:23.747885+0000","last_clean":"2024-09-14T04:33:23.747885+0000","last_became_active":"2024-09-14T04:32:55.316545+0000","last_became_peered":"2024-09-14T04:32:55.316545+0000","last_unstale":"2024-09-14T04:33:23.747885+0000","last_undegraded":"2024-09-14T04:33:23.747885+0000","last_fullsized":"2024-09-14T04:33:23.747885+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:24.194356+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":125,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732129+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:33:23.732129+0000","last_peered":"2024-09-14T04:33:23.732129+0000","last_clean":"2024-09-14T04:33:23.732129+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:33:23.732129+0000","last_undegraded":"2024-09-14T04:33:23.732129+0000","last_fullsized":"2024-09-14T04:33:23.732129+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":809,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732556+0000","last_change":"2024-09-14T04:30:04.291109+0000","last_active":"2024-09-14T04:33:23.732556+0000","last_peered":"2024-09-14T04:33:23.732556+0000","last_clean":"2024-09-14T04:33:23.732556+0000","last_became_active":"2024-09-14T04:30:04.290804+0000","last_became_peered":"2024-09-14T04:30:04.290804+0000","last_unstale":"2024-09-14T04:33:23.732556+0000","last_undegraded":"2024-09-14T04:33:23.732556+0000","last_fullsized":"2024-09-14T04:33:23.732556+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:05.742866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730705+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:33:23.730705+0000","last_peered":"2024-09-14T04:33:23.730705+0000","last_clean":"2024-09-14T04:33:23.730705+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:33:23.730705+0000","last_undegraded":"2024-09-14T04:33:23.730705+0000","last_fullsized":"2024-09-14T04:33:23.730705+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.a","version":"549'14","reported_seq":31,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.737690+0000","last_change":"2024-09-14T04:33:23.738192+0000","last_active":"2024-09-14T04:33:24.737690+0000","last_peered":"2024-09-14T04:33:24.737690+0000","last_clean":"2024-09-14T04:33:24.737690+0000","last_became_active":"2024-09-14T04:33:23.737817+0000","last_became_peered":"2024-09-14T04:33:23.737817+0000","last_unstale":"2024-09-14T04:33:24.737690+0000","last_undegraded":"2024-09-14T04:33:24.737690+0000","last_fullsized":"2024-09-14T04:33:24.737690+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:50.545970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":35,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1238,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.744060+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:33:24.744060+0000","last_peered":"2024-09-14T04:33:24.744060+0000","last_clean":"2024-09-14T04:33:24.744060+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:33:24.744060+0000","last_undegraded":"2024-09-14T04:33:24.744060+0000","last_fullsized":"2024-09-14T04:33:24.744060+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":728,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742129+0000","last_change":"2024-09-14T04:30:04.305324+0000","last_active":"2024-09-14T04:33:24.742129+0000","last_peered":"2024-09-14T04:33:24.742129+0000","last_clean":"2024-09-14T04:33:24.742129+0000","last_became_active":"2024-09-14T04:30:04.289343+0000","last_became_peered":"2024-09-14T04:30:04.289343+0000","last_unstale":"2024-09-14T04:33:24.742129+0000","last_undegraded":"2024-09-14T04:33:24.742129+0000","last_fullsized":"2024-09-14T04:33:24.742129+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:41.011672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":111,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733367+0000","last_change":"2024-09-14T04:29:37.991840+0000","last_active":"2024-09-14T04:33:23.733367+0000","last_peered":"2024-09-14T04:33:23.733367+0000","last_clean":"2024-09-14T04:33:23.733367+0000","last_became_active":"2024-09-14T04:29:37.991503+0000","last_became_peered":"2024-09-14T04:29:37.991503+0000","last_unstale":"2024-09-14T04:33:23.733367+0000","last_undegraded":"2024-09-14T04:33:23.733367+0000","last_fullsized":"2024-09-14T04:33:23.733367+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:22.858456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":547,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747638+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:33:23.747638+0000","last_peered":"2024-09-14T04:33:23.747638+0000","last_clean":"2024-09-14T04:33:23.747638+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:33:23.747638+0000","last_undegraded":"2024-09-14T04:33:23.747638+0000","last_fullsized":"2024-09-14T04:33:23.747638+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.18","version":"549'15","reported_seq":55,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.737093+0000","last_change":"2024-09-14T04:33:23.742623+0000","last_active":"2024-09-14T04:33:24.737093+0000","last_peered":"2024-09-14T04:33:24.737093+0000","last_clean":"2024-09-14T04:33:24.737093+0000","last_became_active":"2024-09-14T04:33:23.742326+0000","last_became_peered":"2024-09-14T04:33:23.742326+0000","last_unstale":"2024-09-14T04:33:24.737093+0000","last_undegraded":"2024-09-14T04:33:24.737093+0000","last_fullsized":"2024-09-14T04:33:24.737093+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:04:35.597727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":86,"num_read_kb":82,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":601,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.749199+0000","last_change":"2024-09-14T04:30:04.291353+0000","last_active":"2024-09-14T04:33:23.749199+0000","last_peered":"2024-09-14T04:33:23.749199+0000","last_clean":"2024-09-14T04:33:23.749199+0000","last_became_active":"2024-09-14T04:30:04.290907+0000","last_became_peered":"2024-09-14T04:30:04.290907+0000","last_unstale":"2024-09-14T04:33:23.749199+0000","last_undegraded":"2024-09-14T04:33:23.749199+0000","last_fullsized":"2024-09-14T04:33:23.749199+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:17.989312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":611,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747392+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:33:23.747392+0000","last_peered":"2024-09-14T04:33:23.747392+0000","last_clean":"2024-09-14T04:33:23.747392+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:33:23.747392+0000","last_undegraded":"2024-09-14T04:33:23.747392+0000","last_fullsized":"2024-09-14T04:33:23.747392+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.19","version":"549'7","reported_seq":20,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324507+0000","last_change":"2024-09-14T04:33:23.753561+0000","last_active":"2024-09-14T04:33:25.324507+0000","last_peered":"2024-09-14T04:33:25.324507+0000","last_clean":"2024-09-14T04:33:25.324507+0000","last_became_active":"2024-09-14T04:33:23.752098+0000","last_became_peered":"2024-09-14T04:33:23.752098+0000","last_unstale":"2024-09-14T04:33:25.324507+0000","last_undegraded":"2024-09-14T04:33:25.324507+0000","last_fullsized":"2024-09-14T04:33:25.324507+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:50:59.459296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":13,"num_read_kb":10,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732484+0000","last_change":"2024-09-14T04:32:52.811129+0000","last_active":"2024-09-14T04:33:23.732484+0000","last_peered":"2024-09-14T04:33:23.732484+0000","last_clean":"2024-09-14T04:33:23.732484+0000","last_became_active":"2024-09-14T04:32:52.810519+0000","last_became_peered":"2024-09-14T04:32:52.810519+0000","last_unstale":"2024-09-14T04:33:23.732484+0000","last_undegraded":"2024-09-14T04:33:23.732484+0000","last_fullsized":"2024-09-14T04:33:23.732484+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:34.831570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":127,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.734955+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:33:23.734955+0000","last_peered":"2024-09-14T04:33:23.734955+0000","last_clean":"2024-09-14T04:33:23.734955+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:33:23.734955+0000","last_undegraded":"2024-09-14T04:33:23.734955+0000","last_fullsized":"2024-09-14T04:33:23.734955+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324496+0000","last_change":"2024-09-14T04:33:23.750270+0000","last_active":"2024-09-14T04:33:25.324496+0000","last_peered":"2024-09-14T04:33:25.324496+0000","last_clean":"2024-09-14T04:33:25.324496+0000","last_became_active":"2024-09-14T04:33:23.749277+0000","last_became_peered":"2024-09-14T04:33:23.749277+0000","last_unstale":"2024-09-14T04:33:25.324496+0000","last_undegraded":"2024-09-14T04:33:25.324496+0000","last_fullsized":"2024-09-14T04:33:25.324496+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:13:27.499363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":1048,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.734194+0000","last_change":"2024-09-14T04:32:55.317954+0000","last_active":"2024-09-14T04:33:23.734194+0000","last_peered":"2024-09-14T04:33:23.734194+0000","last_clean":"2024-09-14T04:33:23.734194+0000","last_became_active":"2024-09-14T04:32:55.317252+0000","last_became_peered":"2024-09-14T04:32:55.317252+0000","last_unstale":"2024-09-14T04:33:23.734194+0000","last_undegraded":"2024-09-14T04:33:23.734194+0000","last_fullsized":"2024-09-14T04:33:23.734194+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:02.900907+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":412,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732883+0000","last_change":"2024-09-14T04:30:04.283436+0000","last_active":"2024-09-14T04:33:23.732883+0000","last_peered":"2024-09-14T04:33:23.732883+0000","last_clean":"2024-09-14T04:33:23.732883+0000","last_became_active":"2024-09-14T04:30:04.283122+0000","last_became_peered":"2024-09-14T04:30:04.283122+0000","last_unstale":"2024-09-14T04:33:23.732883+0000","last_undegraded":"2024-09-14T04:33:23.732883+0000","last_fullsized":"2024-09-14T04:33:23.732883+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.646724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":130,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.731053+0000","last_change":"2024-09-14T04:28:19.257505+0000","last_active":"2024-09-14T04:33:23.731053+0000","last_peered":"2024-09-14T04:33:23.731053+0000","last_clean":"2024-09-14T04:33:23.731053+0000","last_became_active":"2024-09-14T04:28:19.257193+0000","last_became_peered":"2024-09-14T04:28:19.257193+0000","last_unstale":"2024-09-14T04:33:23.731053+0000","last_undegraded":"2024-09-14T04:33:23.731053+0000","last_fullsized":"2024-09-14T04:33:23.731053+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:03:33.146736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.737388+0000","last_change":"2024-09-14T04:33:23.752590+0000","last_active":"2024-09-14T04:33:24.737388+0000","last_peered":"2024-09-14T04:33:24.737388+0000","last_clean":"2024-09-14T04:33:24.737388+0000","last_became_active":"2024-09-14T04:33:23.752280+0000","last_became_peered":"2024-09-14T04:33:23.752280+0000","last_unstale":"2024-09-14T04:33:24.737388+0000","last_undegraded":"2024-09-14T04:33:24.737388+0000","last_fullsized":"2024-09-14T04:33:24.737388+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:18.380012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":712,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742044+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:33:24.742044+0000","last_peered":"2024-09-14T04:33:24.742044+0000","last_clean":"2024-09-14T04:33:24.742044+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:33:24.742044+0000","last_undegraded":"2024-09-14T04:33:24.742044+0000","last_fullsized":"2024-09-14T04:33:24.742044+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":830,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.743508+0000","last_change":"2024-09-14T04:32:52.817482+0000","last_active":"2024-09-14T04:33:24.743508+0000","last_peered":"2024-09-14T04:33:24.743508+0000","last_clean":"2024-09-14T04:33:24.743508+0000","last_became_active":"2024-09-14T04:32:52.817095+0000","last_became_peered":"2024-09-14T04:32:52.817095+0000","last_unstale":"2024-09-14T04:33:24.743508+0000","last_undegraded":"2024-09-14T04:33:24.743508+0000","last_fullsized":"2024-09-14T04:33:24.743508+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:31.086628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.14","version":"549'7","reported_seq":20,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739208+0000","last_change":"2024-09-14T04:33:23.742391+0000","last_active":"2024-09-14T04:33:24.739208+0000","last_peered":"2024-09-14T04:33:24.739208+0000","last_clean":"2024-09-14T04:33:24.739208+0000","last_became_active":"2024-09-14T04:33:23.742095+0000","last_became_peered":"2024-09-14T04:33:23.742095+0000","last_unstale":"2024-09-14T04:33:24.739208+0000","last_undegraded":"2024-09-14T04:33:24.739208+0000","last_fullsized":"2024-09-14T04:33:24.739208+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:40.970678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":26,"num_write":8,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":111,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.733980+0000","last_change":"2024-09-14T04:30:04.283086+0000","last_active":"2024-09-14T04:33:23.733980+0000","last_peered":"2024-09-14T04:33:23.733980+0000","last_clean":"2024-09-14T04:33:23.733980+0000","last_became_active":"2024-09-14T04:30:04.282704+0000","last_became_peered":"2024-09-14T04:30:04.282704+0000","last_unstale":"2024-09-14T04:33:23.733980+0000","last_undegraded":"2024-09-14T04:33:23.733980+0000","last_fullsized":"2024-09-14T04:33:23.733980+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:03.488767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.730435+0000","last_change":"2024-09-14T04:30:04.283463+0000","last_active":"2024-09-14T04:33:23.730435+0000","last_peered":"2024-09-14T04:33:23.730435+0000","last_clean":"2024-09-14T04:33:23.730435+0000","last_became_active":"2024-09-14T04:30:04.283157+0000","last_became_peered":"2024-09-14T04:30:04.283157+0000","last_unstale":"2024-09-14T04:33:23.730435+0000","last_undegraded":"2024-09-14T04:33:23.730435+0000","last_fullsized":"2024-09-14T04:33:23.730435+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:02:11.942507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.15","version":"549'1","reported_seq":13,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.737741+0000","last_change":"2024-09-14T04:33:23.743442+0000","last_active":"2024-09-14T04:33:24.737741+0000","last_peered":"2024-09-14T04:33:24.737741+0000","last_clean":"2024-09-14T04:33:24.737741+0000","last_became_active":"2024-09-14T04:33:23.743124+0000","last_became_peered":"2024-09-14T04:33:23.743124+0000","last_unstale":"2024-09-14T04:33:24.737741+0000","last_undegraded":"2024-09-14T04:33:24.737741+0000","last_fullsized":"2024-09-14T04:33:24.737741+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:28:03.018083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":736,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742005+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:33:24.742005+0000","last_peered":"2024-09-14T04:33:24.742005+0000","last_clean":"2024-09-14T04:33:24.742005+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:33:24.742005+0000","last_undegraded":"2024-09-14T04:33:24.742005+0000","last_fullsized":"2024-09-14T04:33:24.742005+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":761,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.743880+0000","last_change":"2024-09-14T04:30:04.298473+0000","last_active":"2024-09-14T04:33:24.743880+0000","last_peered":"2024-09-14T04:33:24.743880+0000","last_clean":"2024-09-14T04:33:24.743880+0000","last_became_active":"2024-09-14T04:30:04.295176+0000","last_became_peered":"2024-09-14T04:30:04.295176+0000","last_unstale":"2024-09-14T04:33:24.743880+0000","last_undegraded":"2024-09-14T04:33:24.743880+0000","last_fullsized":"2024-09-14T04:33:24.743880+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:02:30.762732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.16","version":"549'22","reported_seq":46,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741385+0000","last_change":"2024-09-14T04:33:23.744660+0000","last_active":"2024-09-14T04:33:24.741385+0000","last_peered":"2024-09-14T04:33:24.741385+0000","last_clean":"2024-09-14T04:33:24.741385+0000","last_became_active":"2024-09-14T04:33:23.744297+0000","last_became_peered":"2024-09-14T04:33:23.744297+0000","last_unstale":"2024-09-14T04:33:24.741385+0000","last_undegraded":"2024-09-14T04:33:24.741385+0000","last_fullsized":"2024-09-14T04:33:24.741385+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:18:24.724676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":44,"num_write":14,"num_write_kb":17,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":291,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.748056+0000","last_change":"2024-09-14T04:32:52.818866+0000","last_active":"2024-09-14T04:33:23.748056+0000","last_peered":"2024-09-14T04:33:23.748056+0000","last_clean":"2024-09-14T04:33:23.748056+0000","last_became_active":"2024-09-14T04:32:52.817822+0000","last_became_peered":"2024-09-14T04:32:52.817822+0000","last_unstale":"2024-09-14T04:33:23.748056+0000","last_undegraded":"2024-09-14T04:33:23.748056+0000","last_fullsized":"2024-09-14T04:33:23.748056+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:56:44.935438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":160,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324688+0000","last_change":"2024-09-14T04:30:04.295082+0000","last_active":"2024-09-14T04:33:25.324688+0000","last_peered":"2024-09-14T04:33:25.324688+0000","last_clean":"2024-09-14T04:33:25.324688+0000","last_became_active":"2024-09-14T04:30:04.293846+0000","last_became_peered":"2024-09-14T04:30:04.293846+0000","last_unstale":"2024-09-14T04:33:25.324688+0000","last_undegraded":"2024-09-14T04:33:25.324688+0000","last_fullsized":"2024-09-14T04:33:25.324688+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:05.873391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.17","version":"549'2","reported_seq":13,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.325395+0000","last_change":"2024-09-14T04:33:23.750547+0000","last_active":"2024-09-14T04:33:25.325395+0000","last_peered":"2024-09-14T04:33:25.325395+0000","last_clean":"2024-09-14T04:33:25.325395+0000","last_became_active":"2024-09-14T04:33:23.749214+0000","last_became_peered":"2024-09-14T04:33:23.749214+0000","last_unstale":"2024-09-14T04:33:25.325395+0000","last_undegraded":"2024-09-14T04:33:25.325395+0000","last_fullsized":"2024-09-14T04:33:25.325395+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.181831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":351,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.741320+0000","last_change":"2024-09-14T04:30:04.293298+0000","last_active":"2024-09-14T04:33:24.741320+0000","last_peered":"2024-09-14T04:33:24.741320+0000","last_clean":"2024-09-14T04:33:24.741320+0000","last_became_active":"2024-09-14T04:30:04.292876+0000","last_became_peered":"2024-09-14T04:30:04.292876+0000","last_unstale":"2024-09-14T04:33:24.741320+0000","last_undegraded":"2024-09-14T04:33:24.741320+0000","last_fullsized":"2024-09-14T04:33:24.741320+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:13.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":740,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742026+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:33:24.742026+0000","last_peered":"2024-09-14T04:33:24.742026+0000","last_clean":"2024-09-14T04:33:24.742026+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:33:24.742026+0000","last_undegraded":"2024-09-14T04:33:24.742026+0000","last_fullsized":"2024-09-14T04:33:24.742026+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.10","version":"549'4","reported_seq":15,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324398+0000","last_change":"2024-09-14T04:33:23.749504+0000","last_active":"2024-09-14T04:33:25.324398+0000","last_peered":"2024-09-14T04:33:25.324398+0000","last_clean":"2024-09-14T04:33:25.324398+0000","last_became_active":"2024-09-14T04:33:23.748997+0000","last_became_peered":"2024-09-14T04:33:23.748997+0000","last_unstale":"2024-09-14T04:33:25.324398+0000","last_undegraded":"2024-09-14T04:33:25.324398+0000","last_fullsized":"2024-09-14T04:33:25.324398+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:48:21.541061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":5,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.750375+0000","last_change":"2024-09-14T04:30:04.296875+0000","last_active":"2024-09-14T04:33:23.750375+0000","last_peered":"2024-09-14T04:33:23.750375+0000","last_clean":"2024-09-14T04:33:23.750375+0000","last_became_active":"2024-09-14T04:30:04.296576+0000","last_became_peered":"2024-09-14T04:30:04.296576+0000","last_unstale":"2024-09-14T04:33:23.750375+0000","last_undegraded":"2024-09-14T04:33:23.750375+0000","last_fullsized":"2024-09-14T04:33:23.750375+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:40.996256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":282,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747151+0000","last_change":"2024-09-14T04:32:52.824597+0000","last_active":"2024-09-14T04:33:23.747151+0000","last_peered":"2024-09-14T04:33:23.747151+0000","last_clean":"2024-09-14T04:33:23.747151+0000","last_became_active":"2024-09-14T04:32:52.818458+0000","last_became_peered":"2024-09-14T04:32:52.818458+0000","last_unstale":"2024-09-14T04:33:23.747151+0000","last_undegraded":"2024-09-14T04:33:23.747151+0000","last_fullsized":"2024-09-14T04:33:23.747151+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:02.830142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"549'5","reported_seq":18,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:25.324550+0000","last_change":"2024-09-14T04:33:23.751376+0000","last_active":"2024-09-14T04:33:25.324550+0000","last_peered":"2024-09-14T04:33:25.324550+0000","last_clean":"2024-09-14T04:33:25.324550+0000","last_became_active":"2024-09-14T04:33:23.751066+0000","last_became_peered":"2024-09-14T04:33:23.751066+0000","last_unstale":"2024-09-14T04:33:25.324550+0000","last_undegraded":"2024-09-14T04:33:25.324550+0000","last_fullsized":"2024-09-14T04:33:25.324550+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:47:09.730963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":162,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747586+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:33:23.747586+0000","last_peered":"2024-09-14T04:33:23.747586+0000","last_clean":"2024-09-14T04:33:23.747586+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:33:23.747586+0000","last_undegraded":"2024-09-14T04:33:23.747586+0000","last_fullsized":"2024-09-14T04:33:23.747586+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":460,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.747789+0000","last_change":"2024-09-14T04:32:52.822812+0000","last_active":"2024-09-14T04:33:23.747789+0000","last_peered":"2024-09-14T04:33:23.747789+0000","last_clean":"2024-09-14T04:33:23.747789+0000","last_became_active":"2024-09-14T04:32:52.817376+0000","last_became_peered":"2024-09-14T04:32:52.817376+0000","last_unstale":"2024-09-14T04:33:23.747789+0000","last_undegraded":"2024-09-14T04:33:23.747789+0000","last_fullsized":"2024-09-14T04:33:23.747789+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:38.165374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.12","version":"549'1","reported_seq":13,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.738046+0000","last_change":"2024-09-14T04:33:23.739629+0000","last_active":"2024-09-14T04:33:24.738046+0000","last_peered":"2024-09-14T04:33:24.738046+0000","last_clean":"2024-09-14T04:33:24.738046+0000","last_became_active":"2024-09-14T04:33:23.739290+0000","last_became_peered":"2024-09-14T04:33:23.739290+0000","last_unstale":"2024-09-14T04:33:24.738046+0000","last_undegraded":"2024-09-14T04:33:24.738046+0000","last_fullsized":"2024-09-14T04:33:24.738046+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:20.605022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":460,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.740893+0000","last_change":"2024-09-14T04:29:38.010269+0000","last_active":"2024-09-14T04:33:24.740893+0000","last_peered":"2024-09-14T04:33:24.740893+0000","last_clean":"2024-09-14T04:33:24.740893+0000","last_became_active":"2024-09-14T04:29:38.009875+0000","last_became_peered":"2024-09-14T04:29:38.009875+0000","last_unstale":"2024-09-14T04:33:24.740893+0000","last_undegraded":"2024-09-14T04:33:24.740893+0000","last_fullsized":"2024-09-14T04:33:24.740893+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:38.314967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":22,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.732457+0000","last_change":"2024-09-14T04:32:52.808518+0000","last_active":"2024-09-14T04:33:23.732457+0000","last_peered":"2024-09-14T04:33:23.732457+0000","last_clean":"2024-09-14T04:33:23.732457+0000","last_became_active":"2024-09-14T04:32:52.807430+0000","last_became_peered":"2024-09-14T04:32:52.807430+0000","last_unstale":"2024-09-14T04:33:23.732457+0000","last_undegraded":"2024-09-14T04:33:23.732457+0000","last_fullsized":"2024-09-14T04:33:23.732457+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:24.095929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":11,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.739420+0000","last_change":"2024-09-14T04:33:23.753894+0000","last_active":"2024-09-14T04:33:24.739420+0000","last_peered":"2024-09-14T04:33:24.739420+0000","last_clean":"2024-09-14T04:33:24.739420+0000","last_became_active":"2024-09-14T04:33:23.753592+0000","last_became_peered":"2024-09-14T04:33:23.753592+0000","last_unstale":"2024-09-14T04:33:24.739420+0000","last_undegraded":"2024-09-14T04:33:24.739420+0000","last_fullsized":"2024-09-14T04:33:24.739420+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":548,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:33:22.721160+0000","last_clean_scrub_stamp":"2024-09-14T04:33:22.721160+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:02:09.296143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":552,"reported_epoch":549,"state":"active+clean","last_fresh":"2024-09-14T04:33:23.746604+0000","last_change":"2024-09-14T04:32:52.818107+0000","last_active":"2024-09-14T04:33:23.746604+0000","last_peered":"2024-09-14T04:33:23.746604+0000","last_clean":"2024-09-14T04:33:23.746604+0000","last_became_active":"2024-09-14T04:32:52.817622+0000","last_became_peered":"2024-09-14T04:32:52.817622+0000","last_unstale":"2024-09-14T04:33:23.746604+0000","last_undegraded":"2024-09-14T04:33:23.746604+0000","last_fullsized":"2024-09-14T04:33:23.746604+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:14.162666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":773,"reported_epoch":550,"state":"active+clean","last_fresh":"2024-09-14T04:33:24.742137+0000","last_change":"2024-09-14T04:29:38.016663+0000","last_active":"2024-09-14T04:33:24.742137+0000","last_peered":"2024-09-14T04:33:24.742137+0000","last_clean":"2024-09-14T04:33:24.742137+0000","last_became_active":"2024-09-14T04:29:38.007007+0000","last_became_peered":"2024-09-14T04:29:38.007007+0000","last_unstale":"2024-09-14T04:33:24.742137+0000","last_undegraded":"2024-09-14T04:33:24.742137+0000","last_fullsized":"2024-09-14T04:33:24.742137+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:55:55.070473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":22506,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":15,"num_pg":32,"stat_sum":{"num_bytes":374,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":1984,"num_read_kb":10073,"num_write":1024,"num_write_kb":5082,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":26,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":814,"ondisk_log_size":814,"up":96,"acting":96,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":399240,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":100,"num_write":25,"num_write_kb":292,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2443447,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201914,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1475732,"kb_used_data":7944,"kb_used_omap":1,"kb_used_meta":1467774,"kb_avail":92269420,"statfs":{"total":95995035648,"available":94483886080,"internally_reserved":0,"allocated":8134656,"data_stored":4517346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1503001035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463552,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213060,"kb_used_data":8504,"kb_used_omap":1,"kb_used_meta":1204542,"kb_avail":92532092,"statfs":{"total":95995035648,"available":94752862208,"internally_reserved":0,"allocated":8708096,"data_stored":5107663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1233451463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834225,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613112,"kb_used_data":8172,"kb_used_omap":5,"kb_used_meta":604922,"kb_avail":93132040,"statfs":{"total":95995035648,"available":95367208960,"internally_reserved":0,"allocated":8368128,"data_stored":4775466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":619440189},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986824,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722472,"kb_used_data":7944,"kb_used_omap":1,"kb_used_meta":1714494,"kb_avail":92022680,"statfs":{"total":95995035648,"available":94231224320,"internally_reserved":0,"allocated":8134656,"data_stored":4517197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1755642310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748853,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1261940,"kb_used_data":8492,"kb_used_omap":4,"kb_used_meta":1253435,"kb_avail":92483212,"statfs":{"total":95995035648,"available":94702809088,"internally_reserved":0,"allocated":8695808,"data_stored":5102764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1283517996},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":1275605287074,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":463856468396,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342511,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000248,"kb_used_data":8216,"kb_used_omap":10,"kb_used_meta":991989,"kb_avail":92744904,"statfs":{"total":95995035648,"available":94970781696,"internally_reserved":0,"allocated":8413184,"data_stored":4831889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":1015796946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":80,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:33:31.509 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:33:31.509 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:33:31.509 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T04:33:31.509 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:33:31.762 INFO:teuthology.orchestra.run.smithi001.stdout:{ 2024-09-14T04:33:31.763 INFO:teuthology.orchestra.run.smithi001.stdout: "size": 20, 2024-09-14T04:33:31.763 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 600, 2024-09-14T04:33:31.763 INFO:teuthology.orchestra.run.smithi001.stdout: "ops": [ 2024-09-14T04:33:31.763 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.763 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272408 12.4 12:25b358f7:::benchmark_data_smithi001_255951_object126748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.96805767800004, 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.99180387999999997, 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.764 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.765 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.765 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272408 2024-09-14T04:33:31.765 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.766 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.767 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.767 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750218+0000", 2024-09-14T04:33:31.767 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.767 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.767 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750219+0000", 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6390000000000001e-06 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.768 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750225+0000", 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.3650000000000003e-06 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.769 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750237+0000", 2024-09-14T04:33:31.770 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2736e-05 2024-09-14T04:33:31.770 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.770 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.770 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.770 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750244+0000", 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.109e-06 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750439+0000", 2024-09-14T04:33:31.771 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000194601 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.750448+0000", 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.8799999999999997e-06 2024-09-14T04:33:31.772 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732347+0000", 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98189893699999997 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.773 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732448+0000", 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101087 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.774 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.732831+0000", 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038299800000000001 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.775 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737544+0000", 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047135329999999998 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741660+0000", 2024-09-14T04:33:31.776 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041152360000000004 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741800+0000", 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013995699999999999 2024-09-14T04:33:31.777 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741974+0000", 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000174733 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.778 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742021+0000", 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7068999999999999e-05 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.779 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.780 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.780 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.780 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272413 12.4 12:200feb18:::benchmark_data_smithi001_255951_object126753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.780 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:33:31.780 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.962785725, 2024-09-14T04:33:31.781 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98673433499999996, 2024-09-14T04:33:31.781 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.781 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.781 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272413 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.782 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.783 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.783 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.783 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:33:31.783 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.783 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755490+0000", 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.784 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.785 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.785 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755491+0000", 2024-09-14T04:33:31.785 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6619999999999999e-06 2024-09-14T04:33:31.786 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.786 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.786 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.786 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755496+0000", 2024-09-14T04:33:31.787 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5220000000000001e-06 2024-09-14T04:33:31.787 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.787 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.787 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755511+0000", 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4866e-05 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.788 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755518+0000", 2024-09-14T04:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.046e-06 2024-09-14T04:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.789 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755570+0000", 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2743e-05 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.755583+0000", 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2396e-05 2024-09-14T04:33:31.790 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.791 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.791 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.791 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735225+0000", 2024-09-14T04:33:31.791 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.97964242499999998 2024-09-14T04:33:31.791 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735341+0000", 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000116228 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.792 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.735690+0000", 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000348871 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.793 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.741614+0000", 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0059240389999999999 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742089+0000", 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00047413899999999999 2024-09-14T04:33:31.794 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742126+0000", 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7982999999999999e-05 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742181+0000", 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4051000000000001e-05 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.795 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742224+0000", 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3364e-05 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272414 12.4 12:202c47cc:::benchmark_data_smithi001_255951_object126754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.96093599899996, 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98523925499999998, 2024-09-14T04:33:31.796 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272414 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.797 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757339+0000", 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757340+0000", 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.102e-06 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.798 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757344+0000", 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.8759999999999998e-06 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757354+0000", 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.7929999999999992e-06 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757359+0000", 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9350000000000002e-06 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.799 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757399+0000", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9684999999999997e-05 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.757404+0000", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1560000000000003e-06 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737575+0000", 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.98017141100000005 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.800 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.737674+0000", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8740000000000007e-05 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.738014+0000", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00033988699999999998 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742033+0000", 2024-09-14T04:33:31.801 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0040193670000000003 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742282+0000", 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00024841899999999999 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.802 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742345+0000", 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3440000000000002e-05 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742480+0000", 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013516099999999999 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.803 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742579+0000", 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8282999999999994e-05 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272475 12.4 12:26c69f4d:::benchmark_data_smithi001_255951_object126815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.804 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:33:31.805 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.81242622900004, 2024-09-14T04:33:31.805 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.84511323999999999, 2024-09-14T04:33:31.805 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.805 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272475 2024-09-14T04:33:31.806 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.807 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.808 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.808 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.808 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905849+0000", 2024-09-14T04:33:31.808 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.808 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905850+0000", 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6300000000000004e-07 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.809 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905854+0000", 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1999999999999996e-06 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905864+0000", 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3980000000000005e-06 2024-09-14T04:33:31.810 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.905870+0000", 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5899999999999996e-06 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.811 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.906017+0000", 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000146754 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:38.906027+0000", 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0481e-05 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.739685+0000", 2024-09-14T04:33:31.812 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.83365730299999996 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.739814+0000", 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012941100000000001 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.813 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.740118+0000", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000304157 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742297+0000", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0021787550000000001 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742652+0000", 2024-09-14T04:33:31.814 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00035510799999999998 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750860+0000", 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0082081570000000006 2024-09-14T04:33:31.815 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.816 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.816 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.816 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750917+0000", 2024-09-14T04:33:31.816 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7090999999999999e-05 2024-09-14T04:33:31.816 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750962+0000", 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4972000000000002e-05 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.817 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272526 12.4 12:235239d5:::benchmark_data_smithi001_255951_object126866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.71409285300001, 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.74694861300000004, 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.818 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272526 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.819 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004182+0000", 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004184+0000", 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9180000000000001e-06 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.820 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004191+0000", 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.0040000000000002e-06 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.821 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004205+0000", 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.34e-05 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004213+0000", 2024-09-14T04:33:31.822 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1359999999999997e-06 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004884+0000", 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00067096400000000002 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.004894+0000", 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.8309999999999996e-06 2024-09-14T04:33:31.823 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742737+0000", 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.73784364099999999 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.742850+0000", 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000113114 2024-09-14T04:33:31.824 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.743167+0000", 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00031664200000000001 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.825 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.750977+0000", 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0078100720000000004 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751016+0000", 2024-09-14T04:33:31.826 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.9014000000000003e-05 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751057+0000", 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1177999999999998e-05 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751098+0000", 2024-09-14T04:33:31.827 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1044999999999998e-05 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.751131+0000", 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.2654000000000003e-05 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:272546 12.4 12:228731de:::benchmark_data_smithi001_255951_object126886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:33:31.828 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 592.68256465599995, 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.723662044, 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 272546 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:33:31.829 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035711+0000", 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035713+0000", 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.0059999999999999e-06 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.830 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035718+0000", 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.0839999999999999e-06 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035732+0000", 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4499e-05 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035739+0000", 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9809999999999997e-06 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.831 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035786+0000", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6733000000000001e-05 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.035794+0000", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7819999999999994e-06 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.746984+0000", 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.71119057500000005 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.832 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.747103+0000", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119184 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.747474+0000", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000370386 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.755439+0000", 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.007964647 2024-09-14T04:33:31.833 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759237+0000", 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0037988399999999999 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759284+0000", 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6878000000000001e-05 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.834 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759331+0000", 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.7261e-05 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:39.759373+0000", 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1187999999999999e-05 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273651 12.14 12:2adccd9f:::benchmark_data_smithi001_255951_object127991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:33:31.835 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 590.91649958799997, 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.95309075099999996, 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273651 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:33:31.836 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801776+0000", 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801777+0000", 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7910000000000001e-06 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.837 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801783+0000", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1080000000000001e-06 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801800+0000", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7554000000000002e-05 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801808+0000", 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.4359999999999996e-06 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.838 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801880+0000", 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.2213999999999997e-05 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.839 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.801889+0000", 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.3999999999999998e-06 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.748500+0000", 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.94661125000000002 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.840 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.748641+0000", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014093800000000001 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.749078+0000", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00043684399999999998 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754302+0000", 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0052233640000000003 2024-09-14T04:33:31.841 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754347+0000", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.5853999999999998e-05 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754530+0000", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00018233400000000001 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754811+0000", 2024-09-14T04:33:31.842 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00028136900000000001 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754866+0000", 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5294999999999998e-05 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273762 12.14 12:2898c893:::benchmark_data_smithi001_255951_object128102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 590.748065204, 2024-09-14T04:33:31.843 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78739233399999997, 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273762 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:33:31.844 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970210+0000", 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970212+0000", 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9120000000000001e-06 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.845 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970218+0000", 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7860000000000002e-06 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970234+0000", 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.6537000000000001e-05 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970243+0000", 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0599999999999997e-06 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.846 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970364+0000", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000120578 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:40.970373+0000", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7129999999999994e-06 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.751673+0000", 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.78130018899999998 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.847 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.751788+0000", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000115554 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.752301+0000", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00051248099999999996 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757408+0000", 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0051068470000000003 2024-09-14T04:33:31.848 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757453+0000", 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4897000000000001e-05 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757496+0000", 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.3368000000000002e-05 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.849 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757553+0000", 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.7448999999999998e-05 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.757602+0000", 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8962999999999999e-05 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.45647.0:273780 12.14 12:2f2a76c0:::benchmark_data_smithi001_255951_object128120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:33:31.850 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 590.71546313199997, 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.76112474900000004, 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.45647", 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/3564075584", 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 273780 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.851 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002812+0000", 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.852 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002814+0000", 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7260000000000001e-06 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002819+0000", 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.203e-06 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.853 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002844+0000", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.5063e-05 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002850+0000", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.857e-06 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002915+0000", 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5147000000000005e-05 2024-09-14T04:33:31.854 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.002925+0000", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0274e-05 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.754907+0000", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.75198182400000002 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.755018+0000", 2024-09-14T04:33:31.855 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011041599999999999 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.755399+0000", 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00038100900000000002 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763712+0000", 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0083134530000000002 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.856 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763760+0000", 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8164000000000002e-05 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763808+0000", 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.799e-05 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.857 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763888+0000", 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9270000000000005e-05 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:23:41.763937+0000", 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9353e-05 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83507 13.c 13:334b35ae:::benchmark_data_smithi001_306460_object83506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:33:31.858 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 446.619194032, 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.23991748900000001, 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83507 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.859 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099081+0000", 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099082+0000", 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9500000000000003e-07 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.860 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099130+0000", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.8446000000000003e-05 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099141+0000", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1229e-05 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099147+0000", 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.8119999999999996e-06 2024-09-14T04:33:31.861 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.103543+0000", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0043959130000000004 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.103756+0000", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00021284400000000001 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.104153+0000", 2024-09-14T04:33:31.862 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000397289 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107757+0000", 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036035279999999999 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.111968+0000", 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0042115579999999998 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.863 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338759+0000", 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.22679080200000001 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338938+0000", 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000179173 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.338999+0000", 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.0399999999999998e-05 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.864 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:83508 13.c 13:3facc2f7:::benchmark_data_smithi001_306460_object83507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 446.61911975300001, 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.240055885, 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:33:31.865 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 83508 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.866 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099156+0000", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.1399999999999997e-07 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099199+0000", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2914000000000003e-05 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099209+0000", 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0232e-05 2024-09-14T04:33:31.867 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.099221+0000", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2272999999999999e-05 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107806+0000", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0085844770000000001 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.107956+0000", 2024-09-14T04:33:31.868 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015019999999999999 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.108108+0000", 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00015174700000000001 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.114878+0000", 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0067705279999999996 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.869 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.115865+0000", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00098683100000000008 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339066+0000", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.223201183 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339167+0000", 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000101035 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.870 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:26:05.339211+0000", 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.4051000000000002e-05 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:215999 13.14 13:2beb661b:::benchmark_data_smithi001_306460_object97755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.666587+0000", 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 313.051688329, 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.70708346300000002, 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.871 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "reached pg", 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 215999 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666587+0000", 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.872 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666587+0000", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666589+0000", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.716e-06 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666595+0000", 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.3239999999999999e-06 2024-09-14T04:33:31.873 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666608+0000", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2872e-05 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666615+0000", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1300000000000003e-06 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666684+0000", 2024-09-14T04:33:31.874 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.9357999999999996e-05 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.666804+0000", 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000119407 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2,5,7", 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.667322+0000", 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00051837699999999999 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.875 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.678726+0000", 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011403623 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.678763+0000", 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.7287999999999999e-05 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.678799+0000", 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.5816999999999999e-05 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.876 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.680347+0000", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0015485519999999999 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.710524+0000", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030177008000000002 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.710632+0000", 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00010767799999999999 2024-09-14T04:33:31.877 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.373583+0000", 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.66295129200000003 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.373670+0000", 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7021000000000006e-05 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.878 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216036 13.14 13:2a672db9:::benchmark_data_smithi001_306460_object97792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:18.722560+0000", 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.99571490800002, 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.65852572200000004, 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216036 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.879 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722560+0000", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722560+0000", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722562+0000", 2024-09-14T04:33:31.880 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.8500000000000001e-06 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722568+0000", 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.6359999999999999e-06 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722580+0000", 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2016e-05 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.881 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722587+0000", 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.5039999999999996e-06 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722639+0000", 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.1149000000000001e-05 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.882 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:18.722647+0000", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.7660000000000004e-06 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.374218+0000", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.65157101100000003 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.374338+0000", 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00011967 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.883 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 0,2,5,6,7", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.374885+0000", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00054658500000000004 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.378351+0000", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0034664119999999999 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.378422+0000", 2024-09-14T04:33:31.884 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1471999999999996e-05 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.378472+0000", 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.9838e-05 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.885 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.378514+0000", 2024-09-14T04:33:31.889 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.2070999999999998e-05 2024-09-14T04:33:31.889 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.378528+0000", 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4068e-05 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.380874+0000", 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0023452809999999998 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.890 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.380942+0000", 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.8690999999999997e-05 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.381086+0000", 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000143702 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46076.0:216081 13.11 13:89044cbd:::benchmark_data_smithi001_306460_object97837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:28:19.389109+0000", 2024-09-14T04:33:31.891 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 312.329166165, 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.463606915, 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46076", 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2029537878", 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 216081 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.892 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389109+0000", 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389109+0000", 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389112+0000", 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.407e-06 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.893 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389118+0000", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.2269999999999998e-06 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389135+0000", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.7244999999999998e-05 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389144+0000", 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 9.0610000000000002e-06 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.894 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389223+0000", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.8701000000000004e-05 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for peered", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:19.389234+0000", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.1389999999999999e-05 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.846428+0000", 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4571935300000001 2024-09-14T04:33:31.895 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.846563+0000", 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000134833 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 2,5,6", 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.847100+0000", 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00053751199999999997 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.896 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.849949+0000", 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0028488350000000001 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.851640+0000", 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001690799 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.851841+0000", 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00020161199999999999 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.897 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.852527+0000", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00068593599999999995 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.852593+0000", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.5417000000000001e-05 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:28:20.852716+0000", 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00012341000000000001 2024-09-14T04:33:31.898 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:95936 14.a 14:5b3b5b84:::benchmark_data_smithi001_337422_object95935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:30:55.835842+0000", 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 155.882432927, 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.20206835000000001, 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.899 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 95936 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835842+0000", 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835842+0000", 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.900 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835845+0000", 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.0149999999999999e-06 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835886+0000", 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.0513999999999999e-05 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.901 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835915+0000", 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 2.9028999999999999e-05 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.835929+0000", 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.4015999999999999e-05 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.836014+0000", 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.5371000000000006e-05 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.902 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op must wait for map", 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.836022+0000", 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.9920000000000007e-06 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.883876+0000", 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.047853795999999997 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.903 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.889181+0000", 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0053047320000000004 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.889669+0000", 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00048785099999999997 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.933062+0000", 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.043393328000000002 2024-09-14T04:33:31.904 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:55.977965+0000", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.044902945 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:56.008023+0000", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.030058148999999999 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:56.035227+0000", 2024-09-14T04:33:31.905 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.027203951000000001 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:56.035430+0000", 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000202766 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:30:56.037911+0000", 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0024808949999999999 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.906 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:157937 14.a 14:50ab7f46:::benchmark_data_smithi001_337422_object58991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:32:10.408161+0000", 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 81.310114244999994, 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.140077337, 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 157937 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.907 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408161+0000", 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408161+0000", 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.908 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408163+0000", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.46e-06 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408168+0000", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.857e-06 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408188+0000", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.959e-05 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.909 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.408197+0000", 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6139999999999992e-06 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.421268+0000", 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.013071685 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.421437+0000", 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00016919799999999999 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.910 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.421593+0000", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000155267 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.430138+0000", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0085456430000000003 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.431148+0000", 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00100982 2024-09-14T04:33:31.911 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.437008+0000", 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0058598679999999998 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.547203+0000", 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.110195247 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.912 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.547240+0000", 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 3.6541000000000001e-05 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.548238+0000", 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00099854700000000002 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.913 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:157946 14.a 14:54babb6e:::benchmark_data_smithi001_337422_object59000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:32:10.414239+0000", 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 81.304036288999995, 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13789143000000001, 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.914 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 157946 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414239+0000", 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414239+0000", 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.915 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414241+0000", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.751e-06 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414246+0000", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.4480000000000002e-06 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414260+0000", 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3892e-05 2024-09-14T04:33:31.916 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.414268+0000", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.2819999999999996e-06 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.425488+0000", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011219685 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.425633+0000", 2024-09-14T04:33:31.917 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00014525200000000001 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.425778+0000", 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.000144547 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.430510+0000", 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0047321280000000004 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.918 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.431944+0000", 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001433794 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.447192+0000", 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015248273 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.551336+0000", 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.10414404100000001 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.919 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.551391+0000", 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.5238999999999997e-05 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:10.552130+0000", 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00073909799999999999 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:172084 14.a 14:58f299f1:::benchmark_data_smithi001_337422_object73138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2024-09-14T04:33:31.920 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:32:23.936419+0000", 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.781856519000002, 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.13967739700000001, 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172084 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.921 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936419+0000", 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936419+0000", 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936420+0000", 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3370000000000001e-06 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.922 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936425+0000", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.955e-06 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936438+0000", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3026e-05 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.936446+0000", 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.0290000000000005e-06 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.923 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.940595+0000", 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0041492409999999997 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.940731+0000", 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00013536700000000001 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.924 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.940812+0000", 2024-09-14T04:33:31.925 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1520999999999994e-05 2024-09-14T04:33:31.925 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.925 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.925 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.948601+0000", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0077885929999999999 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.960934+0000", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.012332856999999999 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.971940+0000", 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.011006648000000001 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.926 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.074924+0000", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.102983246 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.074995+0000", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.1274999999999998e-05 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.076096+0000", 2024-09-14T04:33:31.927 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.001101302 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:172087 14.a 14:56a76261:::benchmark_data_smithi001_337422_object73141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:32:23.937335+0000", 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.780939857999996, 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.140528193, 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.928 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172087 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937335+0000", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937335+0000", 2024-09-14T04:33:31.929 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937337+0000", 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.0979999999999999e-06 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937341+0000", 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 4.6140000000000002e-06 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.930 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937353+0000", 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.2268e-05 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.937361+0000", 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 7.7710000000000006e-06 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.944959+0000", 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0075980470000000001 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.931 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.945045+0000", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.6161000000000004e-05 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.945129+0000", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.4011999999999995e-05 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.948730+0000", 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0036006480000000001 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.932 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.963889+0000", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.015159327 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.971976+0000", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080865050000000008 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.076949+0000", 2024-09-14T04:33:31.933 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.104973211 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.077008+0000", 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.9344e-05 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.077864+0000", 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00085518699999999998 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.934 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "description": "osd_op(client.46352.0:172095 14.a 14:52c94e1d:::benchmark_data_smithi001_337422_object73149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "initiated_at": "2024-09-14T04:32:23.949817+0000", 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "age": 67.768457816999998, 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.14554553200000001, 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "continuous": false, 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "type_data": { 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client_info": { 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client": "client.46352", 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "client_addr": "172.21.15.1:0/2729283375", 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "tid": 172095 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: "events": [ 2024-09-14T04:33:31.935 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "initiated", 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949817+0000", 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "header_read", 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949817+0000", 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.936 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "throttled", 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949819+0000", 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.9709999999999998e-06 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "all_read", 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949833+0000", 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.3875e-05 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.937 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "dispatched", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949848+0000", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 1.5133999999999999e-05 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "queued_for_pg", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949857+0000", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.3450000000000006e-06 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "reached_pg", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949909+0000", 2024-09-14T04:33:31.938 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 5.2405999999999999e-05 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "started", 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.949994+0000", 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.441e-05 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.950075+0000", 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 8.1786000000000006e-05 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "op_commit", 2024-09-14T04:33:31.939 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.953204+0000", 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0031290630000000001 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.963924+0000", 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.010719192000000001 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:23.972001+0000", 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.0080771249999999992 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.940 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.094002+0000", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.122001596 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "commit_sent", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.094069+0000", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 6.6779000000000006e-05 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: }, 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: { 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "event": "done", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "time": "2024-09-14T04:32:24.095363+0000", 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: "duration": 0.00129385 2024-09-14T04:33:31.941 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stdout: } 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stdout: ] 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stdout:} 2024-09-14T04:33:31.942 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:32.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:31 smithi012 ceph-mon[101180]: pgmap v1752: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:32.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:31 smithi001 ceph-mon[107399]: pgmap v1752: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:32.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:31 smithi001 ceph-mon[112165]: pgmap v1752: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:32.656 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:32.675 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:33:33.648 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:34.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:34 smithi001 ceph-mon[107399]: pgmap v1753: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:34.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:34 smithi001 ceph-mon[112165]: pgmap v1753: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:35.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:34 smithi012 ceph-mon[101180]: pgmap v1753: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:35.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:35.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:36.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:36.772 INFO:teuthology.orchestra.run.smithi001.stdout:osd.0: {} 2024-09-14T04:33:36.772 INFO:teuthology.orchestra.run.smithi001.stdout:osd.1: {} 2024-09-14T04:33:36.773 INFO:teuthology.orchestra.run.smithi001.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:33:36.773 INFO:teuthology.orchestra.run.smithi001.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:33:36.773 INFO:teuthology.orchestra.run.smithi001.stdout:osd.2: {} 2024-09-14T04:33:36.774 INFO:teuthology.orchestra.run.smithi001.stdout:osd.3: {} 2024-09-14T04:33:36.774 INFO:teuthology.orchestra.run.smithi001.stdout:osd.4: {} 2024-09-14T04:33:36.774 INFO:teuthology.orchestra.run.smithi001.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:33:36.774 INFO:teuthology.orchestra.run.smithi001.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:33:36.775 INFO:teuthology.orchestra.run.smithi001.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:33:36.775 INFO:teuthology.orchestra.run.smithi001.stdout:osd.5: {} 2024-09-14T04:33:36.775 INFO:teuthology.orchestra.run.smithi001.stdout:osd.6: {} 2024-09-14T04:33:36.775 INFO:teuthology.orchestra.run.smithi001.stdout:osd.7: {} 2024-09-14T04:33:36.776 INFO:teuthology.orchestra.run.smithi001.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:33:36.776 INFO:teuthology.orchestra.run.smithi001.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:33:36.776 INFO:teuthology.orchestra.run.smithi001.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:33:36.833 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:33:36.834 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:33:36.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:36 smithi001 ceph-mon[112165]: pgmap v1754: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:36.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1072398335' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:36 smithi001 ceph-mon[107399]: pgmap v1754: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:36.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1072398335' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:36.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:36] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.51.0" 2024-09-14T04:33:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:36 smithi012 ceph-mon[101180]: pgmap v1754: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1072398335' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:33:38.005 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1754,"stamp":"2024-09-14T04:33:35.656921+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":404813,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":45,"num_read_kb":100,"num_write":89,"num_write_kb":325,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":2465972,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3188,"ondisk_log_size":3188,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7343640,"kb_used_data":49448,"kb_used_omap":25,"kb_used_meta":7293990,"kb_avail":555127272,"statfs":{"total":575970213888,"available":568450326528,"internally_reserved":0,"allocated":50634752,"data_stored":29013743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26477,"internal_metadata":7469045907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374,"num_objects":-34,"num_object_clones":0,"num_object_copies":-102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-34,"num_whiteouts":0,"num_read":-1984,"num_read_kb":-10073,"num_write":-1024,"num_write_kb":-5082,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-26,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.911494"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":912,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093210+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:33:30.093210+0000","last_peered":"2024-09-14T04:33:30.093210+0000","last_clean":"2024-09-14T04:33:30.093210+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:33:30.093210+0000","last_undegraded":"2024-09-14T04:33:30.093210+0000","last_fullsized":"2024-09-14T04:33:30.093210+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:31.029943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":246,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751929+0000","last_change":"2024-09-14T04:30:04.281338+0000","last_active":"2024-09-14T04:33:28.751929+0000","last_peered":"2024-09-14T04:33:28.751929+0000","last_clean":"2024-09-14T04:33:28.751929+0000","last_became_active":"2024-09-14T04:30:04.280363+0000","last_became_peered":"2024-09-14T04:30:04.280363+0000","last_unstale":"2024-09-14T04:33:28.751929+0000","last_undegraded":"2024-09-14T04:33:28.751929+0000","last_fullsized":"2024-09-14T04:33:28.751929+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:50:56.470339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":145,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.752078+0000","last_change":"2024-09-14T04:29:37.998593+0000","last_active":"2024-09-14T04:33:28.752078+0000","last_peered":"2024-09-14T04:33:28.752078+0000","last_clean":"2024-09-14T04:33:28.752078+0000","last_became_active":"2024-09-14T04:29:37.991857+0000","last_became_peered":"2024-09-14T04:29:37.991857+0000","last_unstale":"2024-09-14T04:33:28.752078+0000","last_undegraded":"2024-09-14T04:33:28.752078+0000","last_fullsized":"2024-09-14T04:33:28.752078+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:44:08.110065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":872,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751694+0000","last_change":"2024-09-14T04:32:55.316666+0000","last_active":"2024-09-14T04:33:28.751694+0000","last_peered":"2024-09-14T04:33:28.751694+0000","last_clean":"2024-09-14T04:33:28.751694+0000","last_became_active":"2024-09-14T04:32:55.316301+0000","last_became_peered":"2024-09-14T04:32:55.316301+0000","last_unstale":"2024-09-14T04:33:28.751694+0000","last_undegraded":"2024-09-14T04:33:28.751694+0000","last_fullsized":"2024-09-14T04:33:28.751694+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:15:16.745472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751478+0000","last_change":"2024-09-14T04:29:18.666490+0000","last_active":"2024-09-14T04:33:28.751478+0000","last_peered":"2024-09-14T04:33:28.751478+0000","last_clean":"2024-09-14T04:33:28.751478+0000","last_became_active":"2024-09-14T04:29:18.666113+0000","last_became_peered":"2024-09-14T04:29:18.666113+0000","last_unstale":"2024-09-14T04:33:28.751478+0000","last_undegraded":"2024-09-14T04:33:28.751478+0000","last_fullsized":"2024-09-14T04:33:28.751478+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:30.604568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":735,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093362+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:33:30.093362+0000","last_peered":"2024-09-14T04:33:30.093362+0000","last_clean":"2024-09-14T04:33:30.093362+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:33:30.093362+0000","last_undegraded":"2024-09-14T04:33:30.093362+0000","last_fullsized":"2024-09-14T04:33:30.093362+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":1011,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.096097+0000","last_change":"2024-09-14T04:30:07.304986+0000","last_active":"2024-09-14T04:33:30.096097+0000","last_peered":"2024-09-14T04:33:30.096097+0000","last_clean":"2024-09-14T04:33:30.096097+0000","last_became_active":"2024-09-14T04:30:07.304452+0000","last_became_peered":"2024-09-14T04:30:07.304452+0000","last_unstale":"2024-09-14T04:33:30.096097+0000","last_undegraded":"2024-09-14T04:33:30.096097+0000","last_fullsized":"2024-09-14T04:33:30.096097+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:59.521970+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":721,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097415+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:33:30.097415+0000","last_peered":"2024-09-14T04:33:30.097415+0000","last_clean":"2024-09-14T04:33:30.097415+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:33:30.097415+0000","last_undegraded":"2024-09-14T04:33:30.097415+0000","last_fullsized":"2024-09-14T04:33:30.097415+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":133,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751504+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:33:28.751504+0000","last_peered":"2024-09-14T04:33:28.751504+0000","last_clean":"2024-09-14T04:33:28.751504+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:33:28.751504+0000","last_undegraded":"2024-09-14T04:33:28.751504+0000","last_fullsized":"2024-09-14T04:33:28.751504+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":711,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755462+0000","last_change":"2024-09-14T04:30:07.297746+0000","last_active":"2024-09-14T04:33:28.755462+0000","last_peered":"2024-09-14T04:33:28.755462+0000","last_clean":"2024-09-14T04:33:28.755462+0000","last_became_active":"2024-09-14T04:30:07.297186+0000","last_became_peered":"2024-09-14T04:30:07.297186+0000","last_unstale":"2024-09-14T04:33:28.755462+0000","last_undegraded":"2024-09-14T04:33:28.755462+0000","last_fullsized":"2024-09-14T04:33:28.755462+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:59:40.613090+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":307,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094033+0000","last_change":"2024-09-14T04:30:04.286496+0000","last_active":"2024-09-14T04:33:30.094033+0000","last_peered":"2024-09-14T04:33:30.094033+0000","last_clean":"2024-09-14T04:33:30.094033+0000","last_became_active":"2024-09-14T04:30:04.282753+0000","last_became_peered":"2024-09-14T04:30:04.282753+0000","last_unstale":"2024-09-14T04:33:30.094033+0000","last_undegraded":"2024-09-14T04:33:30.094033+0000","last_fullsized":"2024-09-14T04:33:30.094033+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:39.799809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":115,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755406+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:33:28.755406+0000","last_peered":"2024-09-14T04:33:28.755406+0000","last_clean":"2024-09-14T04:33:28.755406+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:33:28.755406+0000","last_undegraded":"2024-09-14T04:33:28.755406+0000","last_fullsized":"2024-09-14T04:33:28.755406+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":840,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.752462+0000","last_change":"2024-09-14T04:30:07.303908+0000","last_active":"2024-09-14T04:33:28.752462+0000","last_peered":"2024-09-14T04:33:28.752462+0000","last_clean":"2024-09-14T04:33:28.752462+0000","last_became_active":"2024-09-14T04:30:07.302591+0000","last_became_peered":"2024-09-14T04:30:07.302591+0000","last_unstale":"2024-09-14T04:33:28.752462+0000","last_undegraded":"2024-09-14T04:33:28.752462+0000","last_fullsized":"2024-09-14T04:33:28.752462+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.480916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.480916+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:21:51.333058+0000","scrub_duration":0.0047628799999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":128,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.753029+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:33:28.753029+0000","last_peered":"2024-09-14T04:33:28.753029+0000","last_clean":"2024-09-14T04:33:28.753029+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:33:28.753029+0000","last_undegraded":"2024-09-14T04:33:28.753029+0000","last_fullsized":"2024-09-14T04:33:28.753029+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":135,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.750383+0000","last_change":"2024-09-14T04:32:52.814891+0000","last_active":"2024-09-14T04:33:28.750383+0000","last_peered":"2024-09-14T04:33:28.750383+0000","last_clean":"2024-09-14T04:33:28.750383+0000","last_became_active":"2024-09-14T04:32:52.814547+0000","last_became_peered":"2024-09-14T04:32:52.814547+0000","last_unstale":"2024-09-14T04:33:28.750383+0000","last_undegraded":"2024-09-14T04:33:28.750383+0000","last_fullsized":"2024-09-14T04:33:28.750383+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:43:14.342218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1269,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.095090+0000","last_change":"2024-09-14T04:28:22.698300+0000","last_active":"2024-09-14T04:33:30.095090+0000","last_peered":"2024-09-14T04:33:30.095090+0000","last_clean":"2024-09-14T04:33:30.095090+0000","last_became_active":"2024-09-14T04:28:22.697894+0000","last_became_peered":"2024-09-14T04:28:22.697894+0000","last_unstale":"2024-09-14T04:33:30.095090+0000","last_undegraded":"2024-09-14T04:33:30.095090+0000","last_fullsized":"2024-09-14T04:33:30.095090+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:22.747694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:22.747694+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:19.688955+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":141,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.753081+0000","last_change":"2024-09-14T04:30:04.302042+0000","last_active":"2024-09-14T04:33:28.753081+0000","last_peered":"2024-09-14T04:33:28.753081+0000","last_clean":"2024-09-14T04:33:28.753081+0000","last_became_active":"2024-09-14T04:30:04.281337+0000","last_became_peered":"2024-09-14T04:30:04.281337+0000","last_unstale":"2024-09-14T04:33:28.753081+0000","last_undegraded":"2024-09-14T04:33:28.753081+0000","last_fullsized":"2024-09-14T04:33:28.753081+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:39.377188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":600,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.759983+0000","last_change":"2024-09-14T04:32:52.818598+0000","last_active":"2024-09-14T04:33:28.759983+0000","last_peered":"2024-09-14T04:33:28.759983+0000","last_clean":"2024-09-14T04:33:28.759983+0000","last_became_active":"2024-09-14T04:32:52.818281+0000","last_became_peered":"2024-09-14T04:32:52.818281+0000","last_unstale":"2024-09-14T04:33:28.759983+0000","last_undegraded":"2024-09-14T04:33:28.759983+0000","last_fullsized":"2024-09-14T04:33:28.759983+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:39.775021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":504,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756805+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:33:28.756805+0000","last_peered":"2024-09-14T04:33:28.756805+0000","last_clean":"2024-09-14T04:33:28.756805+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:33:28.756805+0000","last_undegraded":"2024-09-14T04:33:28.756805+0000","last_fullsized":"2024-09-14T04:33:28.756805+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":141,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.752540+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:33:28.752540+0000","last_peered":"2024-09-14T04:33:28.752540+0000","last_clean":"2024-09-14T04:33:28.752540+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:33:28.752540+0000","last_undegraded":"2024-09-14T04:33:28.752540+0000","last_fullsized":"2024-09-14T04:33:28.752540+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755548+0000","last_change":"2024-09-14T04:32:52.810817+0000","last_active":"2024-09-14T04:33:28.755548+0000","last_peered":"2024-09-14T04:33:28.755548+0000","last_clean":"2024-09-14T04:33:28.755548+0000","last_became_active":"2024-09-14T04:32:52.809483+0000","last_became_peered":"2024-09-14T04:32:52.809483+0000","last_unstale":"2024-09-14T04:33:28.755548+0000","last_undegraded":"2024-09-14T04:33:28.755548+0000","last_fullsized":"2024-09-14T04:33:28.755548+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:27:00.995883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":561,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.762701+0000","last_change":"2024-09-14T04:27:23.252915+0000","last_active":"2024-09-14T04:33:28.762701+0000","last_peered":"2024-09-14T04:33:28.762701+0000","last_clean":"2024-09-14T04:33:28.762701+0000","last_became_active":"2024-09-14T04:27:23.252443+0000","last_became_peered":"2024-09-14T04:27:23.252443+0000","last_unstale":"2024-09-14T04:33:28.762701+0000","last_undegraded":"2024-09-14T04:33:28.762701+0000","last_fullsized":"2024-09-14T04:33:28.762701+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:26:56.995811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:26:56.995811+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:20.058939+0000","scrub_duration":0.011059105,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":553,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094352+0000","last_change":"2024-09-14T04:29:18.673642+0000","last_active":"2024-09-14T04:33:30.094352+0000","last_peered":"2024-09-14T04:33:30.094352+0000","last_clean":"2024-09-14T04:33:30.094352+0000","last_became_active":"2024-09-14T04:29:18.673365+0000","last_became_peered":"2024-09-14T04:29:18.673365+0000","last_unstale":"2024-09-14T04:33:30.094352+0000","last_undegraded":"2024-09-14T04:33:30.094352+0000","last_fullsized":"2024-09-14T04:33:30.094352+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:32.707692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":715,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097488+0000","last_change":"2024-09-14T04:30:04.292857+0000","last_active":"2024-09-14T04:33:30.097488+0000","last_peered":"2024-09-14T04:33:30.097488+0000","last_clean":"2024-09-14T04:33:30.097488+0000","last_became_active":"2024-09-14T04:30:04.292434+0000","last_became_peered":"2024-09-14T04:30:04.292434+0000","last_unstale":"2024-09-14T04:33:30.097488+0000","last_undegraded":"2024-09-14T04:33:30.097488+0000","last_fullsized":"2024-09-14T04:33:30.097488+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:32:28.993975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":907,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751438+0000","last_change":"2024-09-14T04:29:40.287295+0000","last_active":"2024-09-14T04:33:28.751438+0000","last_peered":"2024-09-14T04:33:28.751438+0000","last_clean":"2024-09-14T04:33:28.751438+0000","last_became_active":"2024-09-14T04:29:40.286891+0000","last_became_peered":"2024-09-14T04:29:40.286891+0000","last_unstale":"2024-09-14T04:33:28.751438+0000","last_undegraded":"2024-09-14T04:33:28.751438+0000","last_fullsized":"2024-09-14T04:33:28.751438+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:13.282637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:13.282637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:59.160582+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":712,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097237+0000","last_change":"2024-09-14T04:10:35.396102+0000","last_active":"2024-09-14T04:33:30.097237+0000","last_peered":"2024-09-14T04:33:30.097237+0000","last_clean":"2024-09-14T04:33:30.097237+0000","last_became_active":"2024-09-14T04:10:35.382904+0000","last_became_peered":"2024-09-14T04:10:35.382904+0000","last_unstale":"2024-09-14T04:33:30.097237+0000","last_undegraded":"2024-09-14T04:33:30.097237+0000","last_fullsized":"2024-09-14T04:33:30.097237+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:39.425341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":593,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761618+0000","last_change":"2024-09-14T04:32:52.819885+0000","last_active":"2024-09-14T04:33:28.761618+0000","last_peered":"2024-09-14T04:33:28.761618+0000","last_clean":"2024-09-14T04:33:28.761618+0000","last_became_active":"2024-09-14T04:32:52.819034+0000","last_became_peered":"2024-09-14T04:32:52.819034+0000","last_unstale":"2024-09-14T04:33:28.761618+0000","last_undegraded":"2024-09-14T04:33:28.761618+0000","last_fullsized":"2024-09-14T04:33:28.761618+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:52.327969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":382,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755219+0000","last_change":"2024-09-14T04:32:59.331716+0000","last_active":"2024-09-14T04:33:28.755219+0000","last_peered":"2024-09-14T04:33:28.755219+0000","last_clean":"2024-09-14T04:33:28.755219+0000","last_became_active":"2024-09-14T04:32:59.331294+0000","last_became_peered":"2024-09-14T04:32:59.331294+0000","last_unstale":"2024-09-14T04:33:28.755219+0000","last_undegraded":"2024-09-14T04:33:28.755219+0000","last_fullsized":"2024-09-14T04:33:28.755219+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:09:34.175191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:09:34.175191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755873+0000","last_change":"2024-09-14T04:30:04.285245+0000","last_active":"2024-09-14T04:33:28.755873+0000","last_peered":"2024-09-14T04:33:28.755873+0000","last_clean":"2024-09-14T04:33:28.755873+0000","last_became_active":"2024-09-14T04:30:04.284576+0000","last_became_peered":"2024-09-14T04:30:04.284576+0000","last_unstale":"2024-09-14T04:33:28.755873+0000","last_undegraded":"2024-09-14T04:33:28.755873+0000","last_fullsized":"2024-09-14T04:33:28.755873+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:54:15.875331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761664+0000","last_change":"2024-09-14T04:30:04.295569+0000","last_active":"2024-09-14T04:33:28.761664+0000","last_peered":"2024-09-14T04:33:28.761664+0000","last_clean":"2024-09-14T04:33:28.761664+0000","last_became_active":"2024-09-14T04:30:04.294571+0000","last_became_peered":"2024-09-14T04:30:04.294571+0000","last_unstale":"2024-09-14T04:33:28.761664+0000","last_undegraded":"2024-09-14T04:33:28.761664+0000","last_fullsized":"2024-09-14T04:33:28.761664+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:40.129021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":957,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097056+0000","last_change":"2024-09-14T04:32:55.317365+0000","last_active":"2024-09-14T04:33:30.097056+0000","last_peered":"2024-09-14T04:33:30.097056+0000","last_clean":"2024-09-14T04:33:30.097056+0000","last_became_active":"2024-09-14T04:32:55.316994+0000","last_became_peered":"2024-09-14T04:32:55.316994+0000","last_unstale":"2024-09-14T04:33:30.097056+0000","last_undegraded":"2024-09-14T04:33:30.097056+0000","last_fullsized":"2024-09-14T04:33:30.097056+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.965705+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.965705+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:17.732488+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.759769+0000","last_change":"2024-09-14T04:30:04.291481+0000","last_active":"2024-09-14T04:33:28.759769+0000","last_peered":"2024-09-14T04:33:28.759769+0000","last_clean":"2024-09-14T04:33:28.759769+0000","last_became_active":"2024-09-14T04:30:04.291034+0000","last_became_peered":"2024-09-14T04:30:04.291034+0000","last_unstale":"2024-09-14T04:33:28.759769+0000","last_undegraded":"2024-09-14T04:33:28.759769+0000","last_fullsized":"2024-09-14T04:33:28.759769+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:31:09.399818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":590,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.763359+0000","last_change":"2024-09-14T04:32:52.817581+0000","last_active":"2024-09-14T04:33:28.763359+0000","last_peered":"2024-09-14T04:33:28.763359+0000","last_clean":"2024-09-14T04:33:28.763359+0000","last_became_active":"2024-09-14T04:32:52.817276+0000","last_became_peered":"2024-09-14T04:32:52.817276+0000","last_unstale":"2024-09-14T04:33:28.763359+0000","last_undegraded":"2024-09-14T04:33:28.763359+0000","last_fullsized":"2024-09-14T04:33:28.763359+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:15:27.119610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":742,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755351+0000","last_change":"2024-09-14T04:29:40.277117+0000","last_active":"2024-09-14T04:33:28.755351+0000","last_peered":"2024-09-14T04:33:28.755351+0000","last_clean":"2024-09-14T04:33:28.755351+0000","last_became_active":"2024-09-14T04:29:40.276702+0000","last_became_peered":"2024-09-14T04:29:40.276702+0000","last_unstale":"2024-09-14T04:33:28.755351+0000","last_undegraded":"2024-09-14T04:33:28.755351+0000","last_fullsized":"2024-09-14T04:33:28.755351+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:19.375908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:19.375908+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:17:20.933166+0000","scrub_duration":0.0015503489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":307,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093008+0000","last_change":"2024-09-14T04:30:04.298649+0000","last_active":"2024-09-14T04:33:30.093008+0000","last_peered":"2024-09-14T04:33:30.093008+0000","last_clean":"2024-09-14T04:33:30.093008+0000","last_became_active":"2024-09-14T04:30:04.295253+0000","last_became_peered":"2024-09-14T04:30:04.295253+0000","last_unstale":"2024-09-14T04:33:30.093008+0000","last_undegraded":"2024-09-14T04:33:30.093008+0000","last_fullsized":"2024-09-14T04:33:30.093008+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:23.430095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":193,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.762731+0000","last_change":"2024-09-14T04:30:04.296998+0000","last_active":"2024-09-14T04:33:28.762731+0000","last_peered":"2024-09-14T04:33:28.762731+0000","last_clean":"2024-09-14T04:33:28.762731+0000","last_became_active":"2024-09-14T04:30:04.296685+0000","last_became_peered":"2024-09-14T04:30:04.296685+0000","last_unstale":"2024-09-14T04:33:28.762731+0000","last_undegraded":"2024-09-14T04:33:28.762731+0000","last_fullsized":"2024-09-14T04:33:28.762731+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:40:31.564351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":963,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751592+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:33:28.751592+0000","last_peered":"2024-09-14T04:33:28.751592+0000","last_clean":"2024-09-14T04:33:28.751592+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:33:28.751592+0000","last_undegraded":"2024-09-14T04:33:28.751592+0000","last_fullsized":"2024-09-14T04:33:28.751592+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":454,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751978+0000","last_change":"2024-09-14T04:29:38.001028+0000","last_active":"2024-09-14T04:33:28.751978+0000","last_peered":"2024-09-14T04:33:28.751978+0000","last_clean":"2024-09-14T04:33:28.751978+0000","last_became_active":"2024-09-14T04:29:37.987180+0000","last_became_peered":"2024-09-14T04:29:37.987180+0000","last_unstale":"2024-09-14T04:33:28.751978+0000","last_undegraded":"2024-09-14T04:33:28.751978+0000","last_fullsized":"2024-09-14T04:33:28.751978+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:29.081173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":154,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755651+0000","last_change":"2024-09-14T04:29:37.992219+0000","last_active":"2024-09-14T04:33:28.755651+0000","last_peered":"2024-09-14T04:33:28.755651+0000","last_clean":"2024-09-14T04:33:28.755651+0000","last_became_active":"2024-09-14T04:29:37.991324+0000","last_became_peered":"2024-09-14T04:29:37.991324+0000","last_unstale":"2024-09-14T04:33:28.755651+0000","last_undegraded":"2024-09-14T04:33:28.755651+0000","last_fullsized":"2024-09-14T04:33:28.755651+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:08.893407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756306+0000","last_change":"2024-09-14T04:32:52.817519+0000","last_active":"2024-09-14T04:33:28.756306+0000","last_peered":"2024-09-14T04:33:28.756306+0000","last_clean":"2024-09-14T04:33:28.756306+0000","last_became_active":"2024-09-14T04:32:52.817207+0000","last_became_peered":"2024-09-14T04:32:52.817207+0000","last_unstale":"2024-09-14T04:33:28.756306+0000","last_undegraded":"2024-09-14T04:33:28.756306+0000","last_fullsized":"2024-09-14T04:33:28.756306+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:43.967076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":390,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755697+0000","last_change":"2024-09-14T04:32:55.318369+0000","last_active":"2024-09-14T04:33:28.755697+0000","last_peered":"2024-09-14T04:33:28.755697+0000","last_clean":"2024-09-14T04:33:28.755697+0000","last_became_active":"2024-09-14T04:32:55.317929+0000","last_became_peered":"2024-09-14T04:32:55.317929+0000","last_unstale":"2024-09-14T04:33:28.755697+0000","last_undegraded":"2024-09-14T04:33:28.755697+0000","last_fullsized":"2024-09-14T04:33:28.755697+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.938752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.938752+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":134,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.750657+0000","last_change":"2024-09-14T04:30:04.283624+0000","last_active":"2024-09-14T04:33:28.750657+0000","last_peered":"2024-09-14T04:33:28.750657+0000","last_clean":"2024-09-14T04:33:28.750657+0000","last_became_active":"2024-09-14T04:30:04.283298+0000","last_became_peered":"2024-09-14T04:30:04.283298+0000","last_unstale":"2024-09-14T04:33:28.750657+0000","last_undegraded":"2024-09-14T04:33:28.750657+0000","last_fullsized":"2024-09-14T04:33:28.750657+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:32.022169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":958,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093109+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:33:30.093109+0000","last_peered":"2024-09-14T04:33:30.093109+0000","last_clean":"2024-09-14T04:33:30.093109+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:33:30.093109+0000","last_undegraded":"2024-09-14T04:33:30.093109+0000","last_fullsized":"2024-09-14T04:33:30.093109+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":733,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097507+0000","last_change":"2024-09-14T04:30:04.285113+0000","last_active":"2024-09-14T04:33:30.097507+0000","last_peered":"2024-09-14T04:33:30.097507+0000","last_clean":"2024-09-14T04:33:30.097507+0000","last_became_active":"2024-09-14T04:30:04.283783+0000","last_became_peered":"2024-09-14T04:30:04.283783+0000","last_unstale":"2024-09-14T04:33:30.097507+0000","last_undegraded":"2024-09-14T04:33:30.097507+0000","last_fullsized":"2024-09-14T04:33:30.097507+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:13.747229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":821,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:29.759817+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:33:29.759817+0000","last_peered":"2024-09-14T04:33:29.759817+0000","last_clean":"2024-09-14T04:33:29.759817+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:33:29.759817+0000","last_undegraded":"2024-09-14T04:33:29.759817+0000","last_fullsized":"2024-09-14T04:33:29.759817+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":526,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093314+0000","last_change":"2024-09-14T04:30:04.285510+0000","last_active":"2024-09-14T04:33:30.093314+0000","last_peered":"2024-09-14T04:33:30.093314+0000","last_clean":"2024-09-14T04:33:30.093314+0000","last_became_active":"2024-09-14T04:30:04.285170+0000","last_became_peered":"2024-09-14T04:30:04.285170+0000","last_unstale":"2024-09-14T04:33:30.093314+0000","last_undegraded":"2024-09-14T04:33:30.093314+0000","last_fullsized":"2024-09-14T04:33:30.093314+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:32.780039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755276+0000","last_change":"2024-09-14T04:32:52.815265+0000","last_active":"2024-09-14T04:33:28.755276+0000","last_peered":"2024-09-14T04:33:28.755276+0000","last_clean":"2024-09-14T04:33:28.755276+0000","last_became_active":"2024-09-14T04:32:52.814764+0000","last_became_peered":"2024-09-14T04:32:52.814764+0000","last_unstale":"2024-09-14T04:33:28.755276+0000","last_undegraded":"2024-09-14T04:33:28.755276+0000","last_fullsized":"2024-09-14T04:33:28.755276+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:59.683211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":485,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094522+0000","last_change":"2024-09-14T04:30:04.287957+0000","last_active":"2024-09-14T04:33:30.094522+0000","last_peered":"2024-09-14T04:33:30.094522+0000","last_clean":"2024-09-14T04:33:30.094522+0000","last_became_active":"2024-09-14T04:30:04.287313+0000","last_became_peered":"2024-09-14T04:30:04.287313+0000","last_unstale":"2024-09-14T04:33:30.094522+0000","last_undegraded":"2024-09-14T04:33:30.094522+0000","last_fullsized":"2024-09-14T04:33:30.094522+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:25.063506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":705,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093423+0000","last_change":"2024-09-14T04:29:40.285341+0000","last_active":"2024-09-14T04:33:30.093423+0000","last_peered":"2024-09-14T04:33:30.093423+0000","last_clean":"2024-09-14T04:33:30.093423+0000","last_became_active":"2024-09-14T04:29:40.284955+0000","last_became_peered":"2024-09-14T04:29:40.284955+0000","last_unstale":"2024-09-14T04:33:30.093423+0000","last_undegraded":"2024-09-14T04:33:30.093423+0000","last_fullsized":"2024-09-14T04:33:30.093423+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:18:51.087521+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":29,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755929+0000","last_change":"2024-09-14T04:32:57.968797+0000","last_active":"2024-09-14T04:33:28.755929+0000","last_peered":"2024-09-14T04:33:28.755929+0000","last_clean":"2024-09-14T04:33:28.755929+0000","last_became_active":"2024-09-14T04:32:52.810067+0000","last_became_peered":"2024-09-14T04:32:52.810067+0000","last_unstale":"2024-09-14T04:33:28.755929+0000","last_undegraded":"2024-09-14T04:33:28.755929+0000","last_fullsized":"2024-09-14T04:33:28.755929+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:19.561540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":584,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.760592+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:33:28.760592+0000","last_peered":"2024-09-14T04:33:28.760592+0000","last_clean":"2024-09-14T04:33:28.760592+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:33:28.760592+0000","last_undegraded":"2024-09-14T04:33:28.760592+0000","last_fullsized":"2024-09-14T04:33:28.760592+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756247+0000","last_change":"2024-09-14T04:32:52.807897+0000","last_active":"2024-09-14T04:33:28.756247+0000","last_peered":"2024-09-14T04:33:28.756247+0000","last_clean":"2024-09-14T04:33:28.756247+0000","last_became_active":"2024-09-14T04:32:52.806944+0000","last_became_peered":"2024-09-14T04:32:52.806944+0000","last_unstale":"2024-09-14T04:33:28.756247+0000","last_undegraded":"2024-09-14T04:33:28.756247+0000","last_fullsized":"2024-09-14T04:33:28.756247+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:23.347741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":948,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756511+0000","last_change":"2024-09-14T04:32:55.315948+0000","last_active":"2024-09-14T04:33:28.756511+0000","last_peered":"2024-09-14T04:33:28.756511+0000","last_clean":"2024-09-14T04:33:28.756511+0000","last_became_active":"2024-09-14T04:32:55.315594+0000","last_became_peered":"2024-09-14T04:32:55.315594+0000","last_unstale":"2024-09-14T04:33:28.756511+0000","last_undegraded":"2024-09-14T04:33:28.756511+0000","last_fullsized":"2024-09-14T04:33:28.756511+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:18.774218+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:18.774218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:59.189391+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":575,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.092610+0000","last_change":"2024-09-14T04:29:37.992555+0000","last_active":"2024-09-14T04:33:30.092610+0000","last_peered":"2024-09-14T04:33:30.092610+0000","last_clean":"2024-09-14T04:33:30.092610+0000","last_became_active":"2024-09-14T04:29:37.992147+0000","last_became_peered":"2024-09-14T04:29:37.992147+0000","last_unstale":"2024-09-14T04:33:30.092610+0000","last_undegraded":"2024-09-14T04:33:30.092610+0000","last_fullsized":"2024-09-14T04:33:30.092610+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:26:44.313121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":886,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094916+0000","last_change":"2024-09-14T04:30:04.287689+0000","last_active":"2024-09-14T04:33:30.094916+0000","last_peered":"2024-09-14T04:33:30.094916+0000","last_clean":"2024-09-14T04:33:30.094916+0000","last_became_active":"2024-09-14T04:30:04.287280+0000","last_became_peered":"2024-09-14T04:30:04.287280+0000","last_unstale":"2024-09-14T04:33:30.094916+0000","last_undegraded":"2024-09-14T04:33:30.094916+0000","last_fullsized":"2024-09-14T04:33:30.094916+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:19.006536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":191,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.760376+0000","last_change":"2024-09-14T04:30:04.294612+0000","last_active":"2024-09-14T04:33:28.760376+0000","last_peered":"2024-09-14T04:33:28.760376+0000","last_clean":"2024-09-14T04:33:28.760376+0000","last_became_active":"2024-09-14T04:30:04.294288+0000","last_became_peered":"2024-09-14T04:30:04.294288+0000","last_unstale":"2024-09-14T04:33:28.760376+0000","last_undegraded":"2024-09-14T04:33:28.760376+0000","last_fullsized":"2024-09-14T04:33:28.760376+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:58.493527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":878,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756471+0000","last_change":"2024-09-14T04:28:22.699361+0000","last_active":"2024-09-14T04:33:28.756471+0000","last_peered":"2024-09-14T04:33:28.756471+0000","last_clean":"2024-09-14T04:33:28.756471+0000","last_became_active":"2024-09-14T04:28:22.699020+0000","last_became_peered":"2024-09-14T04:28:22.699020+0000","last_unstale":"2024-09-14T04:33:28.756471+0000","last_undegraded":"2024-09-14T04:33:28.756471+0000","last_fullsized":"2024-09-14T04:33:28.756471+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:17.613565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:17.613565+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:15:07.313428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":949,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094650+0000","last_change":"2024-09-14T04:30:04.279568+0000","last_active":"2024-09-14T04:33:30.094650+0000","last_peered":"2024-09-14T04:33:30.094650+0000","last_clean":"2024-09-14T04:33:30.094650+0000","last_became_active":"2024-09-14T04:30:04.279227+0000","last_became_peered":"2024-09-14T04:30:04.279227+0000","last_unstale":"2024-09-14T04:33:30.094650+0000","last_undegraded":"2024-09-14T04:33:30.094650+0000","last_fullsized":"2024-09-14T04:33:30.094650+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:46.120838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":125,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:29.762184+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:33:29.762184+0000","last_peered":"2024-09-14T04:33:29.762184+0000","last_clean":"2024-09-14T04:33:29.762184+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:33:29.762184+0000","last_undegraded":"2024-09-14T04:33:29.762184+0000","last_fullsized":"2024-09-14T04:33:29.762184+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":732,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097435+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:33:30.097435+0000","last_peered":"2024-09-14T04:33:30.097435+0000","last_clean":"2024-09-14T04:33:30.097435+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:33:30.097435+0000","last_undegraded":"2024-09-14T04:33:30.097435+0000","last_fullsized":"2024-09-14T04:33:30.097435+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":895,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761250+0000","last_change":"2024-09-14T04:32:55.317272+0000","last_active":"2024-09-14T04:33:28.761250+0000","last_peered":"2024-09-14T04:33:28.761250+0000","last_clean":"2024-09-14T04:33:28.761250+0000","last_became_active":"2024-09-14T04:32:55.316545+0000","last_became_peered":"2024-09-14T04:32:55.316545+0000","last_unstale":"2024-09-14T04:33:28.761250+0000","last_undegraded":"2024-09-14T04:33:28.761250+0000","last_fullsized":"2024-09-14T04:33:28.761250+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.500980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.500980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:42:24.194356+0000","scrub_duration":0.0039945179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":128,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.753133+0000","last_change":"2024-09-14T04:27:20.362844+0000","last_active":"2024-09-14T04:33:28.753133+0000","last_peered":"2024-09-14T04:33:28.753133+0000","last_clean":"2024-09-14T04:33:28.753133+0000","last_became_active":"2024-09-14T04:27:20.248626+0000","last_became_peered":"2024-09-14T04:27:20.248626+0000","last_unstale":"2024-09-14T04:33:28.753133+0000","last_undegraded":"2024-09-14T04:33:28.753133+0000","last_fullsized":"2024-09-14T04:33:28.753133+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:17.039764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":812,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756316+0000","last_change":"2024-09-14T04:30:04.291109+0000","last_active":"2024-09-14T04:33:28.756316+0000","last_peered":"2024-09-14T04:33:28.756316+0000","last_clean":"2024-09-14T04:33:28.756316+0000","last_became_active":"2024-09-14T04:30:04.290804+0000","last_became_peered":"2024-09-14T04:30:04.290804+0000","last_unstale":"2024-09-14T04:33:28.756316+0000","last_undegraded":"2024-09-14T04:33:28.756316+0000","last_fullsized":"2024-09-14T04:33:28.756316+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:05.742866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":125,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751789+0000","last_change":"2024-09-14T04:27:20.253543+0000","last_active":"2024-09-14T04:33:28.751789+0000","last_peered":"2024-09-14T04:33:28.751789+0000","last_clean":"2024-09-14T04:33:28.751789+0000","last_became_active":"2024-09-14T04:27:20.253218+0000","last_became_peered":"2024-09-14T04:27:20.253218+0000","last_unstale":"2024-09-14T04:33:28.751789+0000","last_undegraded":"2024-09-14T04:33:28.751789+0000","last_fullsized":"2024-09-14T04:33:28.751789+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:41.923204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1245,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.095036+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:33:30.095036+0000","last_peered":"2024-09-14T04:33:30.095036+0000","last_clean":"2024-09-14T04:33:30.095036+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:33:30.095036+0000","last_undegraded":"2024-09-14T04:33:30.095036+0000","last_fullsized":"2024-09-14T04:33:30.095036+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:48.801509+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":734,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097313+0000","last_change":"2024-09-14T04:30:04.305324+0000","last_active":"2024-09-14T04:33:30.097313+0000","last_peered":"2024-09-14T04:33:30.097313+0000","last_clean":"2024-09-14T04:33:30.097313+0000","last_became_active":"2024-09-14T04:30:04.289343+0000","last_became_peered":"2024-09-14T04:30:04.289343+0000","last_unstale":"2024-09-14T04:33:30.097313+0000","last_undegraded":"2024-09-14T04:33:30.097313+0000","last_fullsized":"2024-09-14T04:33:30.097313+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:41.011672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756762+0000","last_change":"2024-09-14T04:29:37.991840+0000","last_active":"2024-09-14T04:33:28.756762+0000","last_peered":"2024-09-14T04:33:28.756762+0000","last_clean":"2024-09-14T04:33:28.756762+0000","last_became_active":"2024-09-14T04:29:37.991503+0000","last_became_peered":"2024-09-14T04:29:37.991503+0000","last_unstale":"2024-09-14T04:33:28.756762+0000","last_undegraded":"2024-09-14T04:33:28.756762+0000","last_fullsized":"2024-09-14T04:33:28.756762+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:27:22.858456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":550,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.760068+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:33:28.760068+0000","last_peered":"2024-09-14T04:33:28.760068+0000","last_clean":"2024-09-14T04:33:28.760068+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:33:28.760068+0000","last_undegraded":"2024-09-14T04:33:28.760068+0000","last_fullsized":"2024-09-14T04:33:28.760068+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":604,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761798+0000","last_change":"2024-09-14T04:30:04.291353+0000","last_active":"2024-09-14T04:33:28.761798+0000","last_peered":"2024-09-14T04:33:28.761798+0000","last_clean":"2024-09-14T04:33:28.761798+0000","last_became_active":"2024-09-14T04:30:04.290907+0000","last_became_peered":"2024-09-14T04:30:04.290907+0000","last_unstale":"2024-09-14T04:33:28.761798+0000","last_undegraded":"2024-09-14T04:33:28.761798+0000","last_fullsized":"2024-09-14T04:33:28.761798+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:17.989312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":614,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761594+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:33:28.761594+0000","last_peered":"2024-09-14T04:33:28.761594+0000","last_clean":"2024-09-14T04:33:28.761594+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:33:28.761594+0000","last_undegraded":"2024-09-14T04:33:28.761594+0000","last_fullsized":"2024-09-14T04:33:28.761594+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.755764+0000","last_change":"2024-09-14T04:32:52.811129+0000","last_active":"2024-09-14T04:33:28.755764+0000","last_peered":"2024-09-14T04:33:28.755764+0000","last_clean":"2024-09-14T04:33:28.755764+0000","last_became_active":"2024-09-14T04:32:52.810519+0000","last_became_peered":"2024-09-14T04:32:52.810519+0000","last_unstale":"2024-09-14T04:33:28.755764+0000","last_undegraded":"2024-09-14T04:33:28.755764+0000","last_fullsized":"2024-09-14T04:33:28.755764+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:34.831570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":130,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.752309+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:33:28.752309+0000","last_peered":"2024-09-14T04:33:28.752309+0000","last_clean":"2024-09-14T04:33:28.752309+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:33:28.752309+0000","last_undegraded":"2024-09-14T04:33:28.752309+0000","last_fullsized":"2024-09-14T04:33:28.752309+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":1051,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751277+0000","last_change":"2024-09-14T04:32:55.317954+0000","last_active":"2024-09-14T04:33:28.751277+0000","last_peered":"2024-09-14T04:33:28.751277+0000","last_clean":"2024-09-14T04:33:28.751277+0000","last_became_active":"2024-09-14T04:32:55.317252+0000","last_became_peered":"2024-09-14T04:32:55.317252+0000","last_unstale":"2024-09-14T04:33:28.751277+0000","last_undegraded":"2024-09-14T04:33:28.751277+0000","last_fullsized":"2024-09-14T04:33:28.751277+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:02.900907+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":415,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.752722+0000","last_change":"2024-09-14T04:30:04.283436+0000","last_active":"2024-09-14T04:33:28.752722+0000","last_peered":"2024-09-14T04:33:28.752722+0000","last_clean":"2024-09-14T04:33:28.752722+0000","last_became_active":"2024-09-14T04:30:04.283122+0000","last_became_peered":"2024-09-14T04:30:04.283122+0000","last_unstale":"2024-09-14T04:33:28.752722+0000","last_undegraded":"2024-09-14T04:33:28.752722+0000","last_fullsized":"2024-09-14T04:33:28.752722+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.646724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":133,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751341+0000","last_change":"2024-09-14T04:28:19.257505+0000","last_active":"2024-09-14T04:33:28.751341+0000","last_peered":"2024-09-14T04:33:28.751341+0000","last_clean":"2024-09-14T04:33:28.751341+0000","last_became_active":"2024-09-14T04:28:19.257193+0000","last_became_peered":"2024-09-14T04:28:19.257193+0000","last_unstale":"2024-09-14T04:33:28.751341+0000","last_undegraded":"2024-09-14T04:33:28.751341+0000","last_fullsized":"2024-09-14T04:33:28.751341+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:03:33.146736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":718,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094617+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:33:30.094617+0000","last_peered":"2024-09-14T04:33:30.094617+0000","last_clean":"2024-09-14T04:33:30.094617+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:33:30.094617+0000","last_undegraded":"2024-09-14T04:33:30.094617+0000","last_fullsized":"2024-09-14T04:33:30.094617+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":836,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093777+0000","last_change":"2024-09-14T04:32:52.817482+0000","last_active":"2024-09-14T04:33:30.093777+0000","last_peered":"2024-09-14T04:33:30.093777+0000","last_clean":"2024-09-14T04:33:30.093777+0000","last_became_active":"2024-09-14T04:32:52.817095+0000","last_became_peered":"2024-09-14T04:32:52.817095+0000","last_unstale":"2024-09-14T04:33:30.093777+0000","last_undegraded":"2024-09-14T04:33:30.093777+0000","last_fullsized":"2024-09-14T04:33:30.093777+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:31.086628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756911+0000","last_change":"2024-09-14T04:30:04.283086+0000","last_active":"2024-09-14T04:33:28.756911+0000","last_peered":"2024-09-14T04:33:28.756911+0000","last_clean":"2024-09-14T04:33:28.756911+0000","last_became_active":"2024-09-14T04:30:04.282704+0000","last_became_peered":"2024-09-14T04:30:04.282704+0000","last_unstale":"2024-09-14T04:33:28.756911+0000","last_undegraded":"2024-09-14T04:33:28.756911+0000","last_fullsized":"2024-09-14T04:33:28.756911+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:03.488767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":55,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.751235+0000","last_change":"2024-09-14T04:30:04.283463+0000","last_active":"2024-09-14T04:33:28.751235+0000","last_peered":"2024-09-14T04:33:28.751235+0000","last_clean":"2024-09-14T04:33:28.751235+0000","last_became_active":"2024-09-14T04:30:04.283157+0000","last_became_peered":"2024-09-14T04:30:04.283157+0000","last_unstale":"2024-09-14T04:33:28.751235+0000","last_undegraded":"2024-09-14T04:33:28.751235+0000","last_fullsized":"2024-09-14T04:33:28.751235+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:02:11.942507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":742,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.097221+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:33:30.097221+0000","last_peered":"2024-09-14T04:33:30.097221+0000","last_clean":"2024-09-14T04:33:30.097221+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:33:30.097221+0000","last_undegraded":"2024-09-14T04:33:30.097221+0000","last_fullsized":"2024-09-14T04:33:30.097221+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":767,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094142+0000","last_change":"2024-09-14T04:30:04.298473+0000","last_active":"2024-09-14T04:33:30.094142+0000","last_peered":"2024-09-14T04:33:30.094142+0000","last_clean":"2024-09-14T04:33:30.094142+0000","last_became_active":"2024-09-14T04:30:04.295176+0000","last_became_peered":"2024-09-14T04:30:04.295176+0000","last_unstale":"2024-09-14T04:33:30.094142+0000","last_undegraded":"2024-09-14T04:33:30.094142+0000","last_fullsized":"2024-09-14T04:33:30.094142+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:02:30.762732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":294,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761491+0000","last_change":"2024-09-14T04:32:52.818866+0000","last_active":"2024-09-14T04:33:28.761491+0000","last_peered":"2024-09-14T04:33:28.761491+0000","last_clean":"2024-09-14T04:33:28.761491+0000","last_became_active":"2024-09-14T04:32:52.817822+0000","last_became_peered":"2024-09-14T04:32:52.817822+0000","last_unstale":"2024-09-14T04:33:28.761491+0000","last_undegraded":"2024-09-14T04:33:28.761491+0000","last_fullsized":"2024-09-14T04:33:28.761491+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:56:44.935438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":163,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.759989+0000","last_change":"2024-09-14T04:30:04.295082+0000","last_active":"2024-09-14T04:33:28.759989+0000","last_peered":"2024-09-14T04:33:28.759989+0000","last_clean":"2024-09-14T04:33:28.759989+0000","last_became_active":"2024-09-14T04:30:04.293846+0000","last_became_peered":"2024-09-14T04:30:04.293846+0000","last_unstale":"2024-09-14T04:33:28.759989+0000","last_undegraded":"2024-09-14T04:33:28.759989+0000","last_fullsized":"2024-09-14T04:33:28.759989+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:05.873391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":357,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094756+0000","last_change":"2024-09-14T04:30:04.293298+0000","last_active":"2024-09-14T04:33:30.094756+0000","last_peered":"2024-09-14T04:33:30.094756+0000","last_clean":"2024-09-14T04:33:30.094756+0000","last_became_active":"2024-09-14T04:30:04.292876+0000","last_became_peered":"2024-09-14T04:30:04.292876+0000","last_unstale":"2024-09-14T04:33:30.094756+0000","last_undegraded":"2024-09-14T04:33:30.094756+0000","last_fullsized":"2024-09-14T04:33:30.094756+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:13.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":746,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.092502+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:33:30.092502+0000","last_peered":"2024-09-14T04:33:30.092502+0000","last_clean":"2024-09-14T04:33:30.092502+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:33:30.092502+0000","last_undegraded":"2024-09-14T04:33:30.092502+0000","last_fullsized":"2024-09-14T04:33:30.092502+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761001+0000","last_change":"2024-09-14T04:30:04.296875+0000","last_active":"2024-09-14T04:33:28.761001+0000","last_peered":"2024-09-14T04:33:28.761001+0000","last_clean":"2024-09-14T04:33:28.761001+0000","last_became_active":"2024-09-14T04:30:04.296576+0000","last_became_peered":"2024-09-14T04:30:04.296576+0000","last_unstale":"2024-09-14T04:33:28.761001+0000","last_undegraded":"2024-09-14T04:33:28.761001+0000","last_fullsized":"2024-09-14T04:33:28.761001+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:24:40.996256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":285,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761294+0000","last_change":"2024-09-14T04:32:52.824597+0000","last_active":"2024-09-14T04:33:28.761294+0000","last_peered":"2024-09-14T04:33:28.761294+0000","last_clean":"2024-09-14T04:33:28.761294+0000","last_became_active":"2024-09-14T04:32:52.818458+0000","last_became_peered":"2024-09-14T04:32:52.818458+0000","last_unstale":"2024-09-14T04:33:28.761294+0000","last_undegraded":"2024-09-14T04:33:28.761294+0000","last_fullsized":"2024-09-14T04:33:28.761294+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:02.830142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":165,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.760414+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:33:28.760414+0000","last_peered":"2024-09-14T04:33:28.760414+0000","last_clean":"2024-09-14T04:33:28.760414+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:33:28.760414+0000","last_undegraded":"2024-09-14T04:33:28.760414+0000","last_fullsized":"2024-09-14T04:33:28.760414+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":463,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.761156+0000","last_change":"2024-09-14T04:32:52.822812+0000","last_active":"2024-09-14T04:33:28.761156+0000","last_peered":"2024-09-14T04:33:28.761156+0000","last_clean":"2024-09-14T04:33:28.761156+0000","last_became_active":"2024-09-14T04:32:52.817376+0000","last_became_peered":"2024-09-14T04:32:52.817376+0000","last_unstale":"2024-09-14T04:33:28.761156+0000","last_undegraded":"2024-09-14T04:33:28.761156+0000","last_fullsized":"2024-09-14T04:33:28.761156+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:38.165374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":466,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.092994+0000","last_change":"2024-09-14T04:29:38.010269+0000","last_active":"2024-09-14T04:33:30.092994+0000","last_peered":"2024-09-14T04:33:30.092994+0000","last_clean":"2024-09-14T04:33:30.092994+0000","last_became_active":"2024-09-14T04:29:38.009875+0000","last_became_peered":"2024-09-14T04:29:38.009875+0000","last_unstale":"2024-09-14T04:33:30.092994+0000","last_undegraded":"2024-09-14T04:33:30.092994+0000","last_fullsized":"2024-09-14T04:33:30.092994+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:38.314967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":25,"reported_epoch":552,"state":"active+clean","last_fresh":"2024-09-14T04:33:28.756342+0000","last_change":"2024-09-14T04:32:52.808518+0000","last_active":"2024-09-14T04:33:28.756342+0000","last_peered":"2024-09-14T04:33:28.756342+0000","last_clean":"2024-09-14T04:33:28.756342+0000","last_became_active":"2024-09-14T04:32:52.807430+0000","last_became_peered":"2024-09-14T04:32:52.807430+0000","last_unstale":"2024-09-14T04:33:28.756342+0000","last_undegraded":"2024-09-14T04:33:28.756342+0000","last_fullsized":"2024-09-14T04:33:28.756342+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:24.095929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":556,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.094447+0000","last_change":"2024-09-14T04:32:52.818107+0000","last_active":"2024-09-14T04:33:30.094447+0000","last_peered":"2024-09-14T04:33:30.094447+0000","last_clean":"2024-09-14T04:33:30.094447+0000","last_became_active":"2024-09-14T04:32:52.817622+0000","last_became_peered":"2024-09-14T04:32:52.817622+0000","last_unstale":"2024-09-14T04:33:30.094447+0000","last_undegraded":"2024-09-14T04:33:30.094447+0000","last_fullsized":"2024-09-14T04:33:30.094447+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:14.162666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":780,"reported_epoch":553,"state":"active+clean","last_fresh":"2024-09-14T04:33:30.093618+0000","last_change":"2024-09-14T04:29:38.016663+0000","last_active":"2024-09-14T04:33:30.093618+0000","last_peered":"2024-09-14T04:33:30.093618+0000","last_clean":"2024-09-14T04:33:30.093618+0000","last_became_active":"2024-09-14T04:29:38.007007+0000","last_became_peered":"2024-09-14T04:29:38.007007+0000","last_unstale":"2024-09-14T04:33:30.093618+0000","last_undegraded":"2024-09-14T04:33:30.093618+0000","last_fullsized":"2024-09-14T04:33:30.093618+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:55:55.070473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":22506,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":399240,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":100,"num_write":25,"num_write_kb":292,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2443447,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201916,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493808,"kb_used_data":7964,"kb_used_omap":1,"kb_used_meta":1485822,"kb_avail":92251344,"statfs":{"total":95995035648,"available":94465376256,"internally_reserved":0,"allocated":8155136,"data_stored":4542806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1521482187},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463554,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213148,"kb_used_data":8536,"kb_used_omap":1,"kb_used_meta":1204542,"kb_avail":92532004,"statfs":{"total":95995035648,"available":94752772096,"internally_reserved":0,"allocated":8740864,"data_stored":5133109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1233451463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834226,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":635284,"kb_used_data":8204,"kb_used_omap":5,"kb_used_meta":627066,"kb_avail":93109868,"statfs":{"total":95995035648,"available":95344504832,"internally_reserved":0,"allocated":8400896,"data_stored":4800912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":642115645},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986825,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722512,"kb_used_data":7956,"kb_used_omap":1,"kb_used_meta":1714494,"kb_avail":92022640,"statfs":{"total":95995035648,"available":94231183360,"internally_reserved":0,"allocated":8146944,"data_stored":4542463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1755642310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748854,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266132,"kb_used_data":8520,"kb_used_omap":4,"kb_used_meta":1257595,"kb_avail":92479020,"statfs":{"total":95995035648,"available":94698516480,"internally_reserved":0,"allocated":8724480,"data_stored":5128304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1287777836},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":1275605287075,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":463856468398,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342513,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012756,"kb_used_data":8268,"kb_used_omap":10,"kb_used_meta":1004469,"kb_avail":92732396,"statfs":{"total":95995035648,"available":94957973504,"internally_reserved":0,"allocated":8466432,"data_stored":4866149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":1028576466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:33:38.011 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:33:38.011 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-09-14T04:33:38.027 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:38.779 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:38 smithi001 ceph-mon[107399]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:38.779 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:38 smithi001 ceph-mon[107399]: pgmap v1755: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:38.780 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:38 smithi001 ceph-mon[112165]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:38.780 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:38 smithi001 ceph-mon[112165]: pgmap v1755: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:38 smithi012 ceph-mon[101180]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:33:39.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:38 smithi012 ceph-mon[101180]: pgmap v1755: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:39.729 INFO:teuthology.orchestra.run.smithi001.stderr:noscrub is unset 2024-09-14T04:33:39.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:39 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/382165590' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:39.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:39 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:39 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/382165590' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:39.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:39 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:39.997 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:39 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/382165590' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:39.997 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:39 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:33:40.573 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:33:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[112165]: pgmap v1756: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[112165]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:33:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[112165]: osdmap e554: 8 total, 8 up, 6 in 2024-09-14T04:33:40.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[107399]: pgmap v1756: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:40.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[107399]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:40.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:33:40.972 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:40 smithi001 ceph-mon[107399]: osdmap e554: 8 total, 8 up, 6 in 2024-09-14T04:33:41.009 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:40 smithi012 ceph-mon[101180]: pgmap v1756: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:41.009 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:40 smithi012 ceph-mon[101180]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:33:41.009 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:40 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:33:41.009 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:40 smithi012 ceph-mon[101180]: osdmap e554: 8 total, 8 up, 6 in 2024-09-14T04:33:41.326 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:41.812 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:41.812 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:41.812 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:42.996 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:42 smithi001 ceph-mon[107399]: pgmap v1758: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:42.997 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:42 smithi001 ceph-mon[112165]: pgmap v1758: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:42 smithi012 ceph-mon[101180]: pgmap v1758: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:43.739 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:33:45.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:44 smithi001 ceph-mon[107399]: pgmap v1759: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:45.052 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:44 smithi001 ceph-mon[112165]: pgmap v1759: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:44 smithi012 ceph-mon[101180]: pgmap v1759: 93 pgs: 93 active+clean; 395 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:45.735 INFO:teuthology.orchestra.run.smithi001.stderr:nodeep-scrub is unset 2024-09-14T04:33:45.840 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[107399]: 1.3 scrub starts 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[107399]: 1.3 scrub ok 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4011068511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[112165]: 1.3 scrub starts 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[112165]: 1.3 scrub ok 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4011068511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:45.841 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:45 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:45 smithi012 ceph-mon[101180]: 1.3 scrub starts 2024-09-14T04:33:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:45 smithi012 ceph-mon[101180]: 1.3 scrub ok 2024-09-14T04:33:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:45 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4011068511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:45 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:33:46.664 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:46.738 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 1 pg_num to 31 2024-09-14T04:33:46.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:46] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.51.0" 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: pgmap v1760: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: osdmap e555: 8 total, 8 up, 6 in 2024-09-14T04:33:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: pgmap v1760: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: osdmap e555: 8 total, 8 up, 6 in 2024-09-14T04:33:46.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: pgmap v1760: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: osdmap e555: 8 total, 8 up, 6 in 2024-09-14T04:33:47.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T04:33:47.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T04:33:47.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[107399]: osdmap e556: 8 total, 8 up, 6 in 2024-09-14T04:33:47.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[107399]: pgmap v1763: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:47.928 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:47.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T04:33:47.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[112165]: osdmap e556: 8 total, 8 up, 6 in 2024-09-14T04:33:47.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[112165]: pgmap v1763: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:47.929 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:47 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3640967976' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T04:33:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:47 smithi012 ceph-mon[101180]: osdmap e556: 8 total, 8 up, 6 in 2024-09-14T04:33:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:47 smithi012 ceph-mon[101180]: pgmap v1763: 93 pgs: 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:47 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:49.007 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:48 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T04:33:49.007 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:48 smithi001 ceph-mon[107399]: osdmap e557: 8 total, 8 up, 6 in 2024-09-14T04:33:49.008 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:48 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T04:33:49.008 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:48 smithi001 ceph-mon[112165]: osdmap e557: 8 total, 8 up, 6 in 2024-09-14T04:33:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:48 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T04:33:49.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:48 smithi012 ceph-mon[101180]: osdmap e557: 8 total, 8 up, 6 in 2024-09-14T04:33:49.278 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:49.278 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:49.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: osdmap e558: 8 total, 8 up, 6 in 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: 1.b scrub starts 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: 1.b scrub ok 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: 1.e scrub starts 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: 1.e scrub ok 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[107399]: pgmap v1766: 103 pgs: 10 unknown, 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: osdmap e558: 8 total, 8 up, 6 in 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: 1.b scrub starts 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: 1.b scrub ok 2024-09-14T04:33:49.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: 1.e scrub starts 2024-09-14T04:33:49.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: 1.e scrub ok 2024-09-14T04:33:49.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:49 smithi001 ceph-mon[112165]: pgmap v1766: 103 pgs: 10 unknown, 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:50.022 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: osdmap e558: 8 total, 8 up, 6 in 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: 1.b scrub starts 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: 1.b scrub ok 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: 1.e scrub starts 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: 1.e scrub ok 2024-09-14T04:33:50.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:49 smithi012 ceph-mon[101180]: pgmap v1766: 103 pgs: 10 unknown, 93 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:50.236 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.9 scrub starts 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.9 scrub ok 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.c scrub starts 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.c scrub ok 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.7 scrub starts 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.7 scrub ok 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.1b scrub starts 2024-09-14T04:33:50.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[107399]: 1.1b scrub ok 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.9 scrub starts 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.9 scrub ok 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.c scrub starts 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.c scrub ok 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.7 scrub starts 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.7 scrub ok 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.1b scrub starts 2024-09-14T04:33:50.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:50 smithi001 ceph-mon[112165]: 1.1b scrub ok 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.9 scrub starts 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.9 scrub ok 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.c scrub starts 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.c scrub ok 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.7 scrub starts 2024-09-14T04:33:51.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.7 scrub ok 2024-09-14T04:33:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:33:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.1b scrub starts 2024-09-14T04:33:51.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:50 smithi012 ceph-mon[101180]: 1.1b scrub ok 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[107399]: 1.1c scrub starts 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[107399]: 1.1c scrub ok 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[107399]: 1.6 scrub starts 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[107399]: 1.6 scrub ok 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[107399]: pgmap v1767: 103 pgs: 1 unknown, 102 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[112165]: 1.1c scrub starts 2024-09-14T04:33:51.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[112165]: 1.1c scrub ok 2024-09-14T04:33:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[112165]: 1.6 scrub starts 2024-09-14T04:33:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[112165]: 1.6 scrub ok 2024-09-14T04:33:51.972 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:51 smithi001 ceph-mon[112165]: pgmap v1767: 103 pgs: 1 unknown, 102 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:51 smithi012 ceph-mon[101180]: 1.1c scrub starts 2024-09-14T04:33:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:51 smithi012 ceph-mon[101180]: 1.1c scrub ok 2024-09-14T04:33:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:51 smithi012 ceph-mon[101180]: 1.6 scrub starts 2024-09-14T04:33:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:51 smithi012 ceph-mon[101180]: 1.6 scrub ok 2024-09-14T04:33:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:51 smithi012 ceph-mon[101180]: pgmap v1767: 103 pgs: 1 unknown, 102 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:52.421 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:52.421 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:52.629 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-14T04:33:52.629 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-09-14T04:33:52.629 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd in 2 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.d scrub starts 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.d scrub ok 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.5 deep-scrub starts 2024-09-14T04:33:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:52 smithi012 ceph-mon[101180]: 1.5 deep-scrub ok 2024-09-14T04:33:53.124 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.d scrub starts 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.d scrub ok 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.5 deep-scrub starts 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[107399]: 1.5 deep-scrub ok 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.d scrub starts 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.d scrub ok 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:33:53.125 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:33:53.126 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.5 deep-scrub starts 2024-09-14T04:33:53.126 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:52 smithi001 ceph-mon[112165]: 1.5 deep-scrub ok 2024-09-14T04:33:53.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:33:53.274 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:53.393 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:33:53.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[107399]: pgmap v1768: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[112165]: pgmap v1768: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:53.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:53 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:33:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:53 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:33:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:53 smithi012 ceph-mon[101180]: pgmap v1768: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T04:33:54.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T04:33:54.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[107399]: osdmap e559: 8 total, 8 up, 6 in 2024-09-14T04:33:54.872 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[107399]: osdmap e560: 8 total, 8 up, 6 in 2024-09-14T04:33:54.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T04:33:54.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[112165]: osdmap e559: 8 total, 8 up, 6 in 2024-09-14T04:33:54.873 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:54 smithi001 ceph-mon[112165]: osdmap e560: 8 total, 8 up, 6 in 2024-09-14T04:33:55.025 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:54 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T04:33:55.025 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:54 smithi012 ceph-mon[101180]: osdmap e559: 8 total, 8 up, 6 in 2024-09-14T04:33:55.025 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:54 smithi012 ceph-mon[101180]: osdmap e560: 8 total, 8 up, 6 in 2024-09-14T04:33:55.260 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:55.260 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:55.320 INFO:teuthology.orchestra.run.smithi001.stderr:marked in osd.2. 2024-09-14T04:33:55.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:33:55.299+0000 7fbfffcba640 -1 mon.a@0(leader).osd e560 definitely_dead 0 2024-09-14T04:33:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:55 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1820571488' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:55 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-14T04:33:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:55 smithi012 ceph-mon[101180]: osdmap e561: 8 total, 8 up, 7 in 2024-09-14T04:33:55.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:55 smithi012 ceph-mon[101180]: pgmap v1772: 103 pgs: 7 peering, 3 remapped+peering, 93 active+clean; 742 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:55.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1820571488' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.957 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[107399]: osdmap e561: 8 total, 8 up, 7 in 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[107399]: pgmap v1772: 103 pgs: 7 peering, 3 remapped+peering, 93 active+clean; 742 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1820571488' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[112165]: osdmap e561: 8 total, 8 up, 7 in 2024-09-14T04:33:55.958 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:55 smithi001 ceph-mon[112165]: pgmap v1772: 103 pgs: 7 peering, 3 remapped+peering, 93 active+clean; 742 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2024-09-14T04:33:55.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:33:55.982 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-09-14T04:33:55.982 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-14T04:33:55.982 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-14T04:33:55.982 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd in 1 2024-09-14T04:33:56.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:33:56.196 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:56.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:56 smithi001 ceph-mon[107399]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:33:56.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:56 smithi001 ceph-mon[107399]: osdmap e562: 8 total, 8 up, 7 in 2024-09-14T04:33:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:33:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:33:56] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.51.0" 2024-09-14T04:33:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:56 smithi001 ceph-mon[112165]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:33:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:56 smithi001 ceph-mon[112165]: osdmap e562: 8 total, 8 up, 7 in 2024-09-14T04:33:57.071 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:56 smithi012 ceph-mon[101180]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:33:57.071 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:56 smithi012 ceph-mon[101180]: osdmap e562: 8 total, 8 up, 7 in 2024-09-14T04:33:57.939 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:33:57.939 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:33:58.327 INFO:teuthology.orchestra.run.smithi001.stderr:marked in osd.1. 2024-09-14T04:33:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:58 smithi012 ceph-mon[101180]: osdmap e563: 8 total, 8 up, 7 in 2024-09-14T04:33:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:58 smithi012 ceph-mon[101180]: pgmap v1775: 103 pgs: 1 remapped, 2 active+undersized+remapped, 14 peering, 3 remapped+peering, 83 active+clean; 640 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9.8 KiB/s rd, 28 KiB/s wr, 6 op/s; 242 KiB/s, 1 objects/s recovering 2024-09-14T04:33:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:58 smithi012 ceph-mon[101180]: 1.1e scrub starts 2024-09-14T04:33:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:58 smithi012 ceph-mon[101180]: 1.1e scrub ok 2024-09-14T04:33:58.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:58 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-14T04:33:58.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[112165]: osdmap e563: 8 total, 8 up, 7 in 2024-09-14T04:33:58.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[112165]: pgmap v1775: 103 pgs: 1 remapped, 2 active+undersized+remapped, 14 peering, 3 remapped+peering, 83 active+clean; 640 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9.8 KiB/s rd, 28 KiB/s wr, 6 op/s; 242 KiB/s, 1 objects/s recovering 2024-09-14T04:33:58.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[112165]: 1.1e scrub starts 2024-09-14T04:33:58.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[112165]: 1.1e scrub ok 2024-09-14T04:33:58.649 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:33:58.234+0000 7fbfffcba640 -1 mon.a@0(leader).osd e563 definitely_dead 0 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[107399]: osdmap e563: 8 total, 8 up, 7 in 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[107399]: pgmap v1775: 103 pgs: 1 remapped, 2 active+undersized+remapped, 14 peering, 3 remapped+peering, 83 active+clean; 640 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 9.8 KiB/s rd, 28 KiB/s wr, 6 op/s; 242 KiB/s, 1 objects/s recovering 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[107399]: 1.1e scrub starts 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[107399]: 1.1e scrub ok 2024-09-14T04:33:58.650 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:58 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-14T04:33:58.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:33:58.873 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:33:59.072 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-14T04:33:59.072 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-14T04:33:59.073 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-09-14T04:33:59.279 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:33:59.332 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.19 scrub starts 2024-09-14T04:33:59.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.19 scrub ok 2024-09-14T04:33:59.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.1c scrub starts 2024-09-14T04:33:59.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.1c scrub ok 2024-09-14T04:33:59.593 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.18 scrub starts 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: 1.18 scrub ok 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[107399]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.19 scrub starts 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.19 scrub ok 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.1c scrub starts 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.1c scrub ok 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:33:59.594 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.18 scrub starts 2024-09-14T04:33:59.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: 1.18 scrub ok 2024-09-14T04:33:59.595 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:33:59 smithi001 ceph-mon[112165]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.19 scrub starts 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.19 scrub ok 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.1c scrub starts 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.1c scrub ok 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/905710304' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.18 scrub starts 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: 1.18 scrub ok 2024-09-14T04:33:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:33:59 smithi012 ceph-mon[101180]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:34:00.534 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: 1.17 scrub starts 2024-09-14T04:34:00.534 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: 1.17 scrub ok 2024-09-14T04:34:00.534 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: pgmap v1778: 103 pgs: 2 active+undersized+remapped, 26 peering, 6 unknown, 2 remapped+peering, 67 active+clean; 640 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 116 KiB/s, 0 objects/s recovering 2024-09-14T04:34:00.534 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:34:00.534 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:34:00.535 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:00 smithi012 ceph-mon[101180]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:34:00.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: 1.17 scrub starts 2024-09-14T04:34:00.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: 1.17 scrub ok 2024-09-14T04:34:00.692 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: pgmap v1778: 103 pgs: 2 active+undersized+remapped, 26 peering, 6 unknown, 2 remapped+peering, 67 active+clean; 640 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 116 KiB/s, 0 objects/s recovering 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[112165]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: 1.17 scrub starts 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: 1.17 scrub ok 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: pgmap v1778: 103 pgs: 2 active+undersized+remapped, 26 peering, 6 unknown, 2 remapped+peering, 67 active+clean; 640 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 116 KiB/s, 0 objects/s recovering 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:34:00.693 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:00 smithi001 ceph-mon[107399]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:34:00.779 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:00.779 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:01.552 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:01 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:34:01.553 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:01 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:34:01.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:34:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:01 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:34:01.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:01 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:34:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:01 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:34:01.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:01 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:34:01.793 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:02.347 INFO:teuthology.orchestra.run.smithi001.stderr:set pool 1 pgp_num to 31 2024-09-14T04:34:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:02 smithi012 ceph-mon[101180]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:34:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T04:34:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:02 smithi012 ceph-mon[101180]: pgmap v1781: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 16 peering, 4 unknown, 1 remapped+peering, 77 active+clean; 497 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 4 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[107399]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[107399]: pgmap v1781: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 16 peering, 4 unknown, 1 remapped+peering, 77 active+clean; 497 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 4 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[112165]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T04:34:02.695 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:02 smithi001 ceph-mon[112165]: pgmap v1781: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 16 peering, 4 unknown, 1 remapped+peering, 77 active+clean; 497 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 4 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:03.113 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-14T04:34:03.330 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:03.347 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T04:34:03.348 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:03 smithi001 ceph-mon[112165]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:34:03.655 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T04:34:03.655 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:03 smithi012 ceph-mon[101180]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:34:03.676 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:03.677 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:03.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3173193609' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T04:34:03.705 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:03 smithi001 ceph-mon[107399]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:34:04.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: 1.3 scrub starts 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: 1.3 scrub ok 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: 1.15 scrub starts 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: 1.15 scrub ok 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: pgmap v1783: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 14 peering, 4 unknown, 1 remapped+peering, 79 active+clean; 599 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[107399]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:34:04.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: 1.3 scrub starts 2024-09-14T04:34:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: 1.3 scrub ok 2024-09-14T04:34:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: 1.15 scrub starts 2024-09-14T04:34:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: 1.15 scrub ok 2024-09-14T04:34:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: pgmap v1783: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 14 peering, 4 unknown, 1 remapped+peering, 79 active+clean; 599 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:04.472 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:04 smithi001 ceph-mon[112165]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:34:04.493 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: 1.3 scrub starts 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: 1.3 scrub ok 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: 1.15 scrub starts 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: 1.15 scrub ok 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: pgmap v1783: 103 pgs: 3 activating+undersized+remapped, 2 active+undersized+remapped, 14 peering, 4 unknown, 1 remapped+peering, 79 active+clean; 599 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-14T04:34:04.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:04 smithi012 ceph-mon[101180]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stdout:mon.a: {} 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stdout:mon.b: {} 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stdout:mon.c: {} 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:34:05.241 INFO:teuthology.orchestra.run.smithi001.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[112165]: 1.16 scrub starts 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[112165]: 1.16 scrub ok 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3337566816' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:34:05.577 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[107399]: 1.16 scrub starts 2024-09-14T04:34:05.578 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[107399]: 1.16 scrub ok 2024-09-14T04:34:05.578 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:05.578 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:05.578 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:05 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3337566816' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:34:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:05 smithi012 ceph-mon[101180]: 1.16 scrub starts 2024-09-14T04:34:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:05 smithi012 ceph-mon[101180]: 1.16 scrub ok 2024-09-14T04:34:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:05.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:05 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3337566816' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:34:06.019 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:34:06.230 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:06.445 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:06 smithi012 ceph-mon[101180]: 1.1d scrub starts 2024-09-14T04:34:06.445 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:06 smithi012 ceph-mon[101180]: 1.1d scrub ok 2024-09-14T04:34:06.445 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:06 smithi012 ceph-mon[101180]: pgmap v1784: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:06.468 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:06.468 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:06.623 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[107399]: 1.1d scrub starts 2024-09-14T04:34:06.623 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[107399]: 1.1d scrub ok 2024-09-14T04:34:06.623 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[107399]: pgmap v1784: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:06.624 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[112165]: 1.1d scrub starts 2024-09-14T04:34:06.624 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[112165]: 1.1d scrub ok 2024-09-14T04:34:06.624 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-mon[112165]: pgmap v1784: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:06] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.51.0" 2024-09-14T04:34:07.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:34:07.288 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:07.645 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:07 smithi001 ceph-mon[107399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-14T04:34:07.645 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:07 smithi001 ceph-mon[107399]: Cluster is now healthy 2024-09-14T04:34:07.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:07 smithi001 ceph-mon[112165]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-14T04:34:07.646 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:07 smithi001 ceph-mon[112165]: Cluster is now healthy 2024-09-14T04:34:07.702 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:07 smithi012 ceph-mon[101180]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2024-09-14T04:34:07.702 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:07 smithi012 ceph-mon[101180]: Cluster is now healthy 2024-09-14T04:34:08.456 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:08 smithi012 ceph-mon[101180]: pgmap v1785: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:08.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:08 smithi001 ceph-mon[107399]: pgmap v1785: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:08.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:08 smithi001 ceph-mon[112165]: pgmap v1785: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:34:08.954 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:08.954 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:09.372 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:34:09.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:34:09.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1901292193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[107399]: 1.1a scrub starts 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[107399]: 1.1a scrub ok 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1901292193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[112165]: 1.1a scrub starts 2024-09-14T04:34:09.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:09 smithi001 ceph-mon[112165]: 1.1a scrub ok 2024-09-14T04:34:09.760 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1901292193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:09 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:09 smithi012 ceph-mon[101180]: 1.1a scrub starts 2024-09-14T04:34:09.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:09 smithi012 ceph-mon[101180]: 1.1a scrub ok 2024-09-14T04:34:10.132 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-14T04:34:10.345 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:10.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:10 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:10.521 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:10 smithi012 ceph-mon[101180]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:34:10.522 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:10 smithi012 ceph-mon[101180]: pgmap v1787: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[112165]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[112165]: pgmap v1787: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[107399]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:34:10.706 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:10 smithi001 ceph-mon[107399]: pgmap v1787: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:11.599 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:11.600 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:12.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:34:12.485 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:12.721 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-14T04:34:12.865 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:12 smithi012 ceph-mon[101180]: pgmap v1788: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:12.865 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:12.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:12 smithi001 ceph-mon[107399]: pgmap v1788: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:12.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:12 smithi001 ceph-mon[112165]: pgmap v1788: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:12.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:34:13.458 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:34:13.666 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:13.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:13 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:13.899 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:13 smithi012 ceph-mon[101180]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:34:14.081 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:13 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:14.081 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:13 smithi001 ceph-mon[107399]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:34:14.081 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:13 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1256857351' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:34:14.081 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:13 smithi001 ceph-mon[112165]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:34:14.132 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:14.133 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:14.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:34:14.894 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:14 smithi001 ceph-mon[107399]: pgmap v1790: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:14.895 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:14 smithi001 ceph-mon[112165]: pgmap v1790: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:14.964 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:15.085 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:14 smithi012 ceph-mon[101180]: pgmap v1790: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:15.739 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:34:16.075 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:16.075 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:15 smithi012 ceph-mon[101180]: pgmap v1791: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:16.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:16.127 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:15 smithi001 ceph-mon[107399]: pgmap v1791: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:16.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:16.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:15 smithi001 ceph-mon[112165]: pgmap v1791: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:16.496 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-14T04:34:16.634 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:16.635 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:16.709 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:16.720 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:16] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.51.0" 2024-09-14T04:34:16.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:16 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:17.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:16 smithi012 ceph-mon[101180]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:34:17.106 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:16 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:17.106 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:16 smithi001 ceph-mon[107399]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:34:17.106 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:16 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4275375049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:17.106 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:16 smithi001 ceph-mon[112165]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:34:17.291 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:34:17.960 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:17 smithi001 ceph-mon[107399]: pgmap v1793: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:17.961 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:17 smithi001 ceph-mon[112165]: pgmap v1793: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:18.031 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:18.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:17 smithi012 ceph-mon[101180]: pgmap v1793: 103 pgs: 103 active+clean; 582 KiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:19.489 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-14T04:34:19.806 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:19 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:19.807 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:19 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:19.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:19 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:34:20.527 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:20.528 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:20.793 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:20 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:20.793 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:20 smithi012 ceph-mon[101180]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:34:20.793 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:20 smithi012 ceph-mon[101180]: pgmap v1795: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:20.793 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:20.793 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:20.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:20.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[112165]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:34:20.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[112165]: pgmap v1795: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:20.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:20.857 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:20.858 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3252145434' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:34:20.858 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[107399]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:34:20.858 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[107399]: pgmap v1795: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:20.858 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:20.858 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:21.297 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:34:21.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:21 smithi001 ceph-mon[107399]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:34:21.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:21 smithi001 ceph-mon[112165]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:34:21.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:21 smithi012 ceph-mon[101180]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:34:21.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:34:22.067 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:22.788 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:22.845 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:22 smithi012 ceph-mon[101180]: 1.19 scrub starts 2024-09-14T04:34:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:22 smithi012 ceph-mon[101180]: 1.19 scrub ok 2024-09-14T04:34:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:22 smithi012 ceph-mon[101180]: pgmap v1797: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:22.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:22 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:22.892 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[112165]: 1.19 scrub starts 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[112165]: 1.19 scrub ok 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[112165]: pgmap v1797: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[107399]: 1.19 scrub starts 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[107399]: 1.19 scrub ok 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[107399]: pgmap v1797: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:22.893 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:22 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:24.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:24 smithi001 ceph-mon[107399]: pgmap v1798: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:24.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:24 smithi001 ceph-mon[112165]: pgmap v1798: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:25.031 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:24 smithi012 ceph-mon[101180]: pgmap v1798: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:25.541 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:25.541 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:26.720 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:26] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.51.0" 2024-09-14T04:34:27.011 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[107399]: pgmap v1799: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:27.011 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.011 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.012 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[112165]: pgmap v1799: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:27.012 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.012 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:26 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.034 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:34:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:26 smithi012 ceph-mon[101180]: pgmap v1799: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:26 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:27.262 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:27.729 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:34:27.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:27 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:27.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:27 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:27 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[107399]: pgmap v1800: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[107399]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[112165]: pgmap v1800: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:28.854 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:28 smithi001 ceph-mon[112165]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:34:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:28 smithi012 ceph-mon[101180]: pgmap v1800: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:28 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2000571894' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:29.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:28 smithi012 ceph-mon[101180]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:34:29.746 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-14T04:34:30.408 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:30.408 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:31.095 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:30 smithi001 ceph-mon[107399]: pgmap v1802: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:31.095 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:30 smithi001 ceph-mon[112165]: pgmap v1802: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:31.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:30 smithi012 ceph-mon[101180]: pgmap v1802: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:31.118 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:34:31.882 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:31.883 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:33.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:32 smithi012 ceph-mon[101180]: pgmap v1803: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:33.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:32 smithi001 ceph-mon[107399]: pgmap v1803: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:33.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:32 smithi001 ceph-mon[112165]: pgmap v1803: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:33.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:33 smithi001 ceph-mon[107399]: pgmap v1804: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:33.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:33 smithi001 ceph-mon[112165]: pgmap v1804: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:34.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:33 smithi012 ceph-mon[101180]: pgmap v1804: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:35.140 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:35.140 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:35.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:35.546 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:35.546 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:36.220 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:34:36.388 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-14T04:34:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: pgmap v1805: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:36.472 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: pgmap v1805: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:34:36.473 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: pgmap v1805: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:34:36.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:34:36.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:36] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-09-14T04:34:37.065 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:34:37.194 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:37.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:37 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:37.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:37 smithi012 ceph-mon[101180]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:34:37.637 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:37 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:37.638 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:37 smithi001 ceph-mon[107399]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:34:37.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:37 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2732140958' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:34:37.638 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:37 smithi001 ceph-mon[112165]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:34:37.976 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:38.376 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:38 smithi001 ceph-mon[107399]: pgmap v1807: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:38.376 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:38 smithi001 ceph-mon[112165]: pgmap v1807: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:38.846 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:38 smithi012 ceph-mon[101180]: pgmap v1807: 103 pgs: 103 active+clean; 582 KiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:40.045 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:40.046 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:40.837 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:40 smithi001 ceph-mon[107399]: pgmap v1808: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:40.837 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:40 smithi001 ceph-mon[112165]: pgmap v1808: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:40 smithi012 ceph-mon[101180]: pgmap v1808: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:41.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:34:41.508 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:41.735 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:34:41.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:41 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1425439186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:41.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:41 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:41 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1425439186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:41.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:41 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:42.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:41 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1425439186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:42.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:41 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:42.489 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-14T04:34:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:42 smithi012 ceph-mon[101180]: pgmap v1809: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:42 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:42 smithi012 ceph-mon[101180]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:34:43.216 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[112165]: pgmap v1809: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:43.217 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:43.217 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[112165]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:34:43.217 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[107399]: pgmap v1809: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:43.217 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:43.217 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:42 smithi001 ceph-mon[107399]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:34:43.510 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:44.688 INFO:teuthology.orchestra.run.smithi001.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:44.688 INFO:teuthology.orchestra.run.smithi001.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:44.942 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:44 smithi001 ceph-mon[107399]: pgmap v1811: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:44.943 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:44 smithi001 ceph-mon[112165]: pgmap v1811: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:45.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:44 smithi012 ceph-mon[101180]: pgmap v1811: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:45.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:34:45.705 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:46.741 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-14T04:34:46.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[107399]: pgmap v1812: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:46.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1782236910' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:46] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-09-14T04:34:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[112165]: pgmap v1812: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1782236910' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:46 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:47.082 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:46 smithi012 ceph-mon[101180]: pgmap v1812: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:47.082 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1782236910' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:47.082 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:46 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:34:47.587 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:34:47.651 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:47.651 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:47.804 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:47.925 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:47.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[107399]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:34:47.926 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[107399]: pgmap v1814: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:47.926 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:47.926 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[112165]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:34:47.926 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:47 smithi001 ceph-mon[112165]: pgmap v1814: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:47.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:47 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:34:47.999 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:47 smithi012 ceph-mon[101180]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:34:48.000 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:47 smithi012 ceph-mon[101180]: pgmap v1814: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:48.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:34:48.491 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:49.764 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:34:49.825 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:49 smithi012 ceph-mon[101180]: pgmap v1815: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:49.825 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:49 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/904020777' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:49.825 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:49 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:50.094 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:50.094 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:50.151 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[112165]: pgmap v1815: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:50.151 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/904020777' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:50.151 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:50.152 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[107399]: pgmap v1815: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:50.152 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/904020777' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:50.152 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:49 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:50.519 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-14T04:34:50.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:34:50.732 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:50.884 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:34:51.006 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:50 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:51.006 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:50 smithi012 ceph-mon[101180]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:34:51.006 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[107399]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[112165]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:34:51.020 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:51.893 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:51 smithi001 ceph-mon[107399]: pgmap v1817: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:51.893 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:51 smithi001 ceph-mon[112165]: pgmap v1817: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:52.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:51 smithi012 ceph-mon[101180]: pgmap v1817: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:52.469 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2024-09-14T04:34:52.470 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:34:52.781 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-14T04:34:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-09-14T04:34:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:52 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3272126991' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.097 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:52 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-09-14T04:34:53.108 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.109 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3272126991' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.109 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.109 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.109 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-09-14T04:34:53.109 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-09-14T04:34:53.110 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3272126991' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.110 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:52 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:34:53.487 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:34:53.703 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:34:54.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:53 smithi012 ceph-mon[101180]: pgmap v1819: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:54.121 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-14T04:34:54.121 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-14T04:34:54.121 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-14T04:34:54.121 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:54.121 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[107399]: pgmap v1819: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:34:54.122 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:53 smithi001 ceph-mon[112165]: pgmap v1819: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:55.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:54 smithi012 ceph-mon[101180]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:34:55.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:54 smithi001 ceph-mon[112165]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:34:55.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:54 smithi001 ceph-mon[107399]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:34:55.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:55 smithi001 ceph-mon[107399]: pgmap v1821: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:55.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:55 smithi001 ceph-mon[112165]: pgmap v1821: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:56.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:55 smithi012 ceph-mon[101180]: pgmap v1821: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:56.794 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:34:56.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:34:56.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:34:56.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:34:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:34:56] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.51.0" 2024-09-14T04:34:57.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:34:57.596 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-14T04:34:57.815 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[107399]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[107399]: pgmap v1823: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[112165]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:34:57.937 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:57 smithi001 ceph-mon[112165]: pgmap v1823: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:57 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/2952341781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:34:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:57 smithi012 ceph-mon[101180]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:34:58.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:57 smithi012 ceph-mon[101180]: pgmap v1823: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:34:59.762 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-14T04:35:00.074 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[107399]: pgmap v1824: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:00.074 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1935560268' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.074 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.074 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[112165]: pgmap v1824: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:00.074 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1935560268' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.075 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:34:59 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:59 smithi012 ceph-mon[101180]: pgmap v1824: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1935560268' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:34:59 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:35:00.491 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:35:00.700 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:01.098 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:00 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:35:01.099 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:00 smithi001 ceph-mon[107399]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:35:01.099 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:00 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:35:01.099 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:00 smithi001 ceph-mon[112165]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:35:01.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:00 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:35:01.099 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:00 smithi012 ceph-mon[101180]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:35:01.833 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:01 smithi001 ceph-mon[107399]: pgmap v1826: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:01.833 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:01 smithi001 ceph-mon[112165]: pgmap v1826: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:02.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:01 smithi012 ceph-mon[101180]: pgmap v1826: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:02.773 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:35:03.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:02 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:03.132 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:02 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:03.132 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:02 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:03.555 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-14T04:35:03.763 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:03.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:03.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[112165]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:35:03.886 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[112165]: pgmap v1828: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:03.886 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:03.887 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[107399]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:35:03.887 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:03 smithi001 ceph-mon[107399]: pgmap v1828: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:03 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1232567776' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:03 smithi012 ceph-mon[101180]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:35:04.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:03 smithi012 ceph-mon[101180]: pgmap v1828: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:05.194 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:05.195 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:05.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:06.028 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-14T04:35:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:06 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/7886502' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:06 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:06 smithi012 ceph-mon[101180]: pgmap v1829: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:06.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/7886502' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.356 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[107399]: pgmap v1829: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:06.356 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/7886502' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:35:06.357 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-mon[112165]: pgmap v1829: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:06.826 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:35:06.970 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:06] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-09-14T04:35:07.038 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:07.314 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:07 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:07.314 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:07 smithi001 ceph-mon[107399]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:35:07.315 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:07 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:07.315 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:07 smithi001 ceph-mon[112165]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:35:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:07 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:35:07.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:07 smithi012 ceph-mon[101180]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:35:08.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:08 smithi001 ceph-mon[107399]: pgmap v1831: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:08.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:08 smithi001 ceph-mon[112165]: pgmap v1831: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:08.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:08 smithi012 ceph-mon[101180]: pgmap v1831: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:09.149 INFO:teuthology.orchestra.run.smithi001.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:35:09.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:09 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:09.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:09 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:09.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:09 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:09.932 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-14T04:35:10.141 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:10.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:10.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[107399]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:35:10.424 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[107399]: pgmap v1833: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:10.424 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:10.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[112165]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:35:10.425 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:10 smithi001 ceph-mon[112165]: pgmap v1833: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:10 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3011544529' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:10 smithi012 ceph-mon[101180]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:35:10.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:10 smithi012 ceph-mon[101180]: pgmap v1833: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:12.740 INFO:teuthology.orchestra.run.smithi001.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-14T04:35:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:12 smithi012 ceph-mon[101180]: pgmap v1834: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:12 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/412864378' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:12 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[107399]: pgmap v1834: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/412864378' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[112165]: pgmap v1834: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/412864378' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.107 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:12 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:35:13.466 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-14T04:35:13.466 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-14T04:35:13.467 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-14T04:35:13.467 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-14T04:35:13.467 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T04:35:13.467 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:35:13.680 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:14.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:13 smithi001 ceph-mon[107399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:14.052 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:13 smithi001 ceph-mon[107399]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:35:14.052 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:13 smithi001 ceph-mon[112165]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:14.052 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:13 smithi001 ceph-mon[112165]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:35:14.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:13 smithi012 ceph-mon[101180]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:35:14.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:13 smithi012 ceph-mon[101180]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[112165]: pgmap v1836: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[112165]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[112165]: 1.17 scrub starts 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[112165]: 1.17 scrub ok 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[107399]: pgmap v1836: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[107399]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:35:15.000 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[107399]: 1.17 scrub starts 2024-09-14T04:35:15.001 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:14 smithi001 ceph-mon[107399]: 1.17 scrub ok 2024-09-14T04:35:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:14 smithi012 ceph-mon[101180]: pgmap v1836: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:14 smithi012 ceph-mon[101180]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:35:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:14 smithi012 ceph-mon[101180]: 1.17 scrub starts 2024-09-14T04:35:15.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:14 smithi012 ceph-mon[101180]: 1.17 scrub ok 2024-09-14T04:35:15.601 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:35:15.601 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":587,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T04:35:13.741586+0000","last_up_change":"2024-09-14T04:24:06.958044+0000","last_in_change":"2024-09-14T04:33:58.322929+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"568","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"557","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":102,"last_clean_end":151,"up_from":173,"up_thru":566,"down_at":152,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6803","nonce":3225184728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6805","nonce":3225184728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6809","nonce":3225184728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6807","nonce":3225184728}]},"public_addr":"172.21.15.1:6803/3225184728","cluster_addr":"172.21.15.1:6805/3225184728","heartbeat_back_addr":"172.21.15.1:6809/3225184728","heartbeat_front_addr":"172.21.15.1:6807/3225184728","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":104,"up_from":108,"up_thru":567,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6811","nonce":2795716422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6813","nonce":2795716422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6817","nonce":2795716422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6815","nonce":2795716422}]},"public_addr":"172.21.15.1:6811/2795716422","cluster_addr":"172.21.15.1:6813/2795716422","heartbeat_back_addr":"172.21.15.1:6817/2795716422","heartbeat_front_addr":"172.21.15.1:6815/2795716422","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":125,"last_clean_end":259,"up_from":297,"up_thru":579,"down_at":260,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6819","nonce":1732438503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6821","nonce":1732438503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6825","nonce":1732438503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6823","nonce":1732438503}]},"public_addr":"172.21.15.1:6819/1732438503","cluster_addr":"172.21.15.1:6821/1732438503","heartbeat_back_addr":"172.21.15.1:6825/1732438503","heartbeat_front_addr":"172.21.15.1:6823/1732438503","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":579,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6827","nonce":3997365471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6829","nonce":3997365471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6833","nonce":3997365471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6831","nonce":3997365471}]},"public_addr":"172.21.15.1:6827/3997365471","cluster_addr":"172.21.15.1:6829/3997365471","heartbeat_back_addr":"172.21.15.1:6833/3997365471","heartbeat_front_addr":"172.21.15.1:6831/3997365471","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":313,"up_from":372,"up_thru":586,"down_at":314,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6809","nonce":3238411616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6811","nonce":3238411616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6815","nonce":3238411616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6813","nonce":3238411616}]},"public_addr":"172.21.15.12:6809/3238411616","cluster_addr":"172.21.15.12:6811/3238411616","heartbeat_back_addr":"172.21.15.12:6815/3238411616","heartbeat_front_addr":"172.21.15.12:6813/3238411616","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":564,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":151},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":259},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":313},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.18","mappings":[{"from":0,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/4171009491":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/3359585252":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.12:0/2389061674":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.12:6834/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2949027530":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/739412744":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/1739684728":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2817796360":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/3674234575":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6801/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/1135126809":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/2685165922":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/41633275":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/416113000":"2024-09-15T03:46:31.869280+0000","172.21.15.12:0/3549839308":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2249577072":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6800/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.12:6835/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:35:15.914 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:15 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1943530016' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:15.915 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:15 smithi001 ceph-mon[112165]: pgmap v1838: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:15.915 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:15 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1943530016' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:15.915 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:15 smithi001 ceph-mon[107399]: pgmap v1838: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:16.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:15 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1943530016' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:16.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:15 smithi012 ceph-mon[101180]: pgmap v1838: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:16.257 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T04:35:16.257 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:35:16.470 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:16.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:16 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:16] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-09-14T04:35:18.538 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:35:18.538 INFO:teuthology.orchestra.run.smithi001.stdout:{"epoch":587,"fsid":"5626d550-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:35:50.494718+0000","modified":"2024-09-14T04:35:13.741586+0000","last_up_change":"2024-09-14T04:24:06.958044+0000","last_in_change":"2024-09-14T04:33:58.322929+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:52.963897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"568","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"557","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:41:17.443254+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:42:33.133167+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:42:34.073494+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86fadbf5-5c8f-4719-b817-f0ff28fb4caa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":102,"last_clean_end":151,"up_from":173,"up_thru":566,"down_at":152,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6802","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6803","nonce":3225184728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6804","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6805","nonce":3225184728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6808","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6809","nonce":3225184728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6806","nonce":3225184728},{"type":"v1","addr":"172.21.15.1:6807","nonce":3225184728}]},"public_addr":"172.21.15.1:6803/3225184728","cluster_addr":"172.21.15.1:6805/3225184728","heartbeat_back_addr":"172.21.15.1:6809/3225184728","heartbeat_front_addr":"172.21.15.1:6807/3225184728","state":["exists","up"]},{"osd":1,"uuid":"cbf0be8f-6a29-4e63-8dca-4d830fcd7d47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":104,"up_from":108,"up_thru":567,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6810","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6811","nonce":2795716422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6812","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6813","nonce":2795716422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6816","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6817","nonce":2795716422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6814","nonce":2795716422},{"type":"v1","addr":"172.21.15.1:6815","nonce":2795716422}]},"public_addr":"172.21.15.1:6811/2795716422","cluster_addr":"172.21.15.1:6813/2795716422","heartbeat_back_addr":"172.21.15.1:6817/2795716422","heartbeat_front_addr":"172.21.15.1:6815/2795716422","state":["exists","up"]},{"osd":2,"uuid":"ca27ddd4-8678-4b03-be41-f1600e007add","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":125,"last_clean_end":259,"up_from":297,"up_thru":579,"down_at":260,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6818","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6819","nonce":1732438503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6820","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6821","nonce":1732438503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6824","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6825","nonce":1732438503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6822","nonce":1732438503},{"type":"v1","addr":"172.21.15.1:6823","nonce":1732438503}]},"public_addr":"172.21.15.1:6819/1732438503","cluster_addr":"172.21.15.1:6821/1732438503","heartbeat_back_addr":"172.21.15.1:6825/1732438503","heartbeat_front_addr":"172.21.15.1:6823/1732438503","state":["exists","up"]},{"osd":3,"uuid":"7004b178-c338-490e-a6f2-fb8510c93c29","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":579,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6826","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6827","nonce":3997365471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6828","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6829","nonce":3997365471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6832","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6833","nonce":3997365471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.1:6830","nonce":3997365471},{"type":"v1","addr":"172.21.15.1:6831","nonce":3997365471}]},"public_addr":"172.21.15.1:6827/3997365471","cluster_addr":"172.21.15.1:6829/3997365471","heartbeat_back_addr":"172.21.15.1:6833/3997365471","heartbeat_front_addr":"172.21.15.1:6831/3997365471","state":["exists","up"]},{"osd":4,"uuid":"40284648-ea27-4db8-977d-cfd9e41c3b7f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6801","nonce":2859222415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6803","nonce":2859222415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6807","nonce":2859222415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":2859222415},{"type":"v1","addr":"172.21.15.12:6805","nonce":2859222415}]},"public_addr":"172.21.15.12:6801/2859222415","cluster_addr":"172.21.15.12:6803/2859222415","heartbeat_back_addr":"172.21.15.12:6807/2859222415","heartbeat_front_addr":"172.21.15.12:6805/2859222415","state":["exists","up"]},{"osd":5,"uuid":"ad6765ec-bbc6-41ce-a1f6-f4581a7a81d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":313,"up_from":372,"up_thru":586,"down_at":314,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6809","nonce":3238411616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6811","nonce":3238411616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6815","nonce":3238411616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":3238411616},{"type":"v1","addr":"172.21.15.12:6813","nonce":3238411616}]},"public_addr":"172.21.15.12:6809/3238411616","cluster_addr":"172.21.15.12:6811/3238411616","heartbeat_back_addr":"172.21.15.12:6815/3238411616","heartbeat_front_addr":"172.21.15.12:6813/3238411616","state":["exists","up"]},{"osd":6,"uuid":"786e0de6-b9fc-4b30-bd1f-a888694aeeab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":564,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6817","nonce":2651831370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6819","nonce":2651831370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6823","nonce":2651831370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2651831370},{"type":"v1","addr":"172.21.15.12:6821","nonce":2651831370}]},"public_addr":"172.21.15.12:6817/2651831370","cluster_addr":"172.21.15.12:6819/2651831370","heartbeat_back_addr":"172.21.15.12:6823/2651831370","heartbeat_front_addr":"172.21.15.12:6821/2651831370","state":["exists","up"]},{"osd":7,"uuid":"409b8e01-c0ad-4f5c-b67a-95b914426b8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6825","nonce":1667024999}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6827","nonce":1667024999}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6831","nonce":1667024999}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1667024999},{"type":"v1","addr":"172.21.15.12:6829","nonce":1667024999}]},"public_addr":"172.21.15.12:6825/1667024999","cluster_addr":"172.21.15.12:6827/1667024999","heartbeat_back_addr":"172.21.15.12:6831/1667024999","heartbeat_front_addr":"172.21.15.12:6829/1667024999","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:02.645982+0000","dead_epoch":151},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:25.864527+0000","dead_epoch":104},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:49.220993+0000","dead_epoch":259},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:13.538735+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:35.089863+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:39:58.914273+0000","dead_epoch":313},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:22.623794+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:45.123526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"3.18","mappings":[{"from":0,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.1:0/2483217146":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/4171009491":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/3359585252":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6801/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/4291653236":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6801/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/1932731734":"2024-09-15T03:36:40.470365+0000","172.21.15.12:0/2389061674":"2024-09-15T03:47:16.144756+0000","172.21.15.1:6800/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.12:6834/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2949027530":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/739412744":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6800/2630182969":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/1739684728":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2817796360":"2024-09-15T03:46:31.869280+0000","172.21.15.1:0/3674234575":"2024-09-15T03:46:31.869280+0000","172.21.15.1:6801/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:6800/2953284694":"2024-09-15T03:36:16.593660+0000","172.21.15.1:6801/1008160518":"2024-09-15T03:41:16.654706+0000","172.21.15.1:6800/226348322":"2024-09-15T03:36:40.470365+0000","172.21.15.1:0/3177706589":"2024-09-15T03:36:16.593660+0000","172.21.15.12:0/1135126809":"2024-09-15T03:47:16.144756+0000","172.21.15.12:0/2685165922":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/4146840439":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/41633275":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2862372200":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/1621027980":"2024-09-15T03:41:16.654706+0000","172.21.15.1:0/416113000":"2024-09-15T03:46:31.869280+0000","172.21.15.12:0/3549839308":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/2249577072":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2358137785":"2024-09-15T03:36:40.470365+0000","172.21.15.1:6800/3323570302":"2024-09-15T03:50:48.967020+0000","172.21.15.1:0/2665058722":"2024-09-15T03:36:16.593660+0000","172.21.15.12:6835/3701332877":"2024-09-15T03:47:16.144756+0000","172.21.15.1:0/3915177672":"2024-09-15T03:41:16.654706+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:35:18.997 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:18 smithi001 ceph-mon[107399]: pgmap v1839: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:18.997 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:18 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3391901658' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:18.998 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:18 smithi001 ceph-mon[112165]: pgmap v1839: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:18.998 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:18 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3391901658' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:18 smithi012 ceph-mon[101180]: pgmap v1839: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:19.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:18 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3391901658' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:35:19.297 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T04:35:19.297 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T04:35:19.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T04:35:19.298 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T04:35:19.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T04:35:19.299 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T04:35:19.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T04:35:19.300 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T04:35:19.667 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.751 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.751 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.758 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.758 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.762 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.764 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:19.770 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:20.922 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:20 smithi001 ceph-mon[107399]: pgmap v1840: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:20.922 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:20 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:20.923 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:20 smithi001 ceph-mon[112165]: pgmap v1840: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:20.923 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:20 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:20 smithi012 ceph-mon[101180]: pgmap v1840: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T04:35:21.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:20 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:22 smithi012 ceph-mon[101180]: pgmap v1841: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-14T04:35:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:22 smithi012 ceph-mon[101180]: 1.8 scrub starts 2024-09-14T04:35:23.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:22 smithi012 ceph-mon[101180]: 1.8 scrub ok 2024-09-14T04:35:23.145 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[112165]: pgmap v1841: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-14T04:35:23.146 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[112165]: 1.8 scrub starts 2024-09-14T04:35:23.146 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[112165]: 1.8 scrub ok 2024-09-14T04:35:23.146 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[107399]: pgmap v1841: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-14T04:35:23.146 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[107399]: 1.8 scrub starts 2024-09-14T04:35:23.146 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:22 smithi001 ceph-mon[107399]: 1.8 scrub ok 2024-09-14T04:35:25.026 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:24 smithi001 ceph-mon[107399]: pgmap v1842: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T04:35:25.026 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:24 smithi001 ceph-mon[112165]: pgmap v1842: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T04:35:25.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:24 smithi012 ceph-mon[101180]: pgmap v1842: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-14T04:35:26.065 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:25 smithi001 ceph-mon[107399]: 1.1b scrub starts 2024-09-14T04:35:26.066 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:25 smithi001 ceph-mon[107399]: 1.1b scrub ok 2024-09-14T04:35:26.066 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:25 smithi001 ceph-mon[112165]: 1.1b scrub starts 2024-09-14T04:35:26.066 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:25 smithi001 ceph-mon[112165]: 1.1b scrub ok 2024-09-14T04:35:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:25 smithi012 ceph-mon[101180]: 1.1b scrub starts 2024-09-14T04:35:26.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:25 smithi012 ceph-mon[101180]: 1.1b scrub ok 2024-09-14T04:35:26.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:26 smithi001 ceph-mon[107399]: pgmap v1843: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:35:26.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:26 smithi001 ceph-mon[112165]: pgmap v1843: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:35:26.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:26 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:26] "GET /metrics HTTP/1.1" 200 34171 "" "Prometheus/2.51.0" 2024-09-14T04:35:27.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:26 smithi012 ceph-mon[101180]: pgmap v1843: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-14T04:35:27.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:27 smithi001 ceph-mon[107399]: pgmap v1844: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:35:27.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:27 smithi001 ceph-mon[112165]: pgmap v1844: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:35:28.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:27 smithi012 ceph-mon[101180]: pgmap v1844: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[107399]: 1.18 deep-scrub starts 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[107399]: 1.18 deep-scrub ok 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[107399]: pgmap v1845: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[112165]: 1.18 deep-scrub starts 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[112165]: 1.18 deep-scrub ok 2024-09-14T04:35:29.902 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:29 smithi001 ceph-mon[112165]: pgmap v1845: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:30.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:29 smithi012 ceph-mon[101180]: 1.18 deep-scrub starts 2024-09-14T04:35:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:29 smithi012 ceph-mon[101180]: 1.18 deep-scrub ok 2024-09-14T04:35:30.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:29 smithi012 ceph-mon[101180]: pgmap v1845: 103 pgs: 103 active+clean; 454 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:32.402 INFO:teuthology.orchestra.run.smithi001.stdout:558345748877 2024-09-14T04:35:32.403 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T04:35:32.942 INFO:teuthology.orchestra.run.smithi001.stdout:236223201940 2024-09-14T04:35:32.943 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T04:35:33.040 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[107399]: pgmap v1846: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:33.040 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[107399]: 1.a scrub starts 2024-09-14T04:35:33.040 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[107399]: 1.a scrub ok 2024-09-14T04:35:33.041 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[112165]: pgmap v1846: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:33.041 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[112165]: 1.a scrub starts 2024-09-14T04:35:33.041 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:32 smithi001 ceph-mon[112165]: 1.a scrub ok 2024-09-14T04:35:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:32 smithi012 ceph-mon[101180]: pgmap v1846: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:32 smithi012 ceph-mon[101180]: 1.a scrub starts 2024-09-14T04:35:33.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:32 smithi012 ceph-mon[101180]: 1.a scrub ok 2024-09-14T04:35:34.878 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:34 smithi001 ceph-mon[107399]: pgmap v1847: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:34.878 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:34 smithi001 ceph-mon[112165]: pgmap v1847: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:35.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:34 smithi012 ceph-mon[101180]: pgmap v1847: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:35.762 INFO:teuthology.orchestra.run.smithi001.stdout:1597727834250 2024-09-14T04:35:35.762 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T04:35:36.077 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:35 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:36.078 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:35 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:36.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:35 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:36.102 INFO:teuthology.orchestra.run.smithi001.stdout:463856468422 2024-09-14T04:35:36.102 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T04:35:36.843 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:36 smithi001 ceph-mon[107399]: pgmap v1848: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:36.843 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:36 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:35:36.843 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:36 smithi001 ceph-mon[112165]: pgmap v1848: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:36.843 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:36 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:35:36.844 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:36 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:36] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-09-14T04:35:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:36 smithi012 ceph-mon[101180]: pgmap v1848: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:37.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:36 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:35:37.222 INFO:teuthology.orchestra.run.smithi001.stdout:133143986850 2024-09-14T04:35:37.222 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T04:35:37.701 INFO:teuthology.orchestra.run.smithi001.stdout:743029342537 2024-09-14T04:35:37.702 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T04:35:37.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:37 smithi001 ceph-mon[107399]: pgmap v1849: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:37.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:37 smithi001 ceph-mon[112165]: pgmap v1849: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:38.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:37 smithi012 ceph-mon[101180]: pgmap v1849: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:38.171 INFO:teuthology.orchestra.run.smithi001.stdout:201863463578 2024-09-14T04:35:38.171 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T04:35:38.715 INFO:teuthology.orchestra.run.smithi001.stdout:1275605287100 2024-09-14T04:35:38.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T04:35:38.856 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.859 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.879 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.883 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.893 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.900 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:38.900 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[107399]: pgmap v1850: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[112165]: pgmap v1850: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:40.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:40 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:40 smithi012 ceph-mon[101180]: pgmap v1850: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:41.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:40 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:35:42.825 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:42 smithi001 ceph-mon[107399]: pgmap v1851: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:42.825 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:42 smithi001 ceph-mon[112165]: pgmap v1851: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:43.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:42 smithi012 ceph-mon[101180]: pgmap v1851: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:44.992 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:44 smithi001 ceph-mon[107399]: pgmap v1852: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:44.993 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:44 smithi001 ceph-mon[112165]: pgmap v1852: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:45.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:44 smithi012 ceph-mon[101180]: pgmap v1852: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:45.944 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:45 smithi001 ceph-mon[112165]: pgmap v1853: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:45.945 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:45 smithi001 ceph-mon[107399]: pgmap v1853: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:46.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:45 smithi012 ceph-mon[101180]: pgmap v1853: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:46.335 INFO:teuthology.orchestra.run.smithi001.stdout:236223201943 2024-09-14T04:35:46.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:46 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3459256783' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:35:46.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:46 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3459256783' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:35:46.971 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:46 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:46] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-09-14T04:35:47.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:46 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3459256783' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:35:47.100 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201940 got 236223201943 for osd.7 2024-09-14T04:35:47.100 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:47.583 INFO:teuthology.orchestra.run.smithi001.stdout:201863463581 2024-09-14T04:35:48.095 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:47 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/995973864' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:35:48.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:47 smithi012 ceph-mon[101180]: pgmap v1854: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:48.152 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:47 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/995973864' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:35:48.152 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:47 smithi001 ceph-mon[112165]: pgmap v1854: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:48.152 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:47 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/995973864' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:35:48.152 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:47 smithi001 ceph-mon[107399]: pgmap v1854: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:48.875 INFO:teuthology.orchestra.run.smithi001.stdout:463856468425 2024-09-14T04:35:48.970 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:48 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3010193055' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:35:48.971 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:48 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3010193055' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:35:49.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:48 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3010193055' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:35:49.938 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:35:50.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:49 smithi001 ceph-mon[112165]: pgmap v1855: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:50.221 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:49 smithi001 ceph-mon[107399]: pgmap v1855: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:50.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:49 smithi012 ceph-mon[101180]: pgmap v1855: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:50.829 INFO:tasks.cephadm.ceph_manager.ceph:need seq 463856468422 got 463856468425 for osd.1 2024-09-14T04:35:50.829 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:51.056 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:50 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:51.056 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:50 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:51.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:50 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:51.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463578 got 201863463581 for osd.6 2024-09-14T04:35:51.963 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:52.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:51 smithi001 ceph-mon[112165]: pgmap v1856: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:52.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:51 smithi001 ceph-mon[107399]: pgmap v1856: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:52.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:51 smithi012 ceph-mon[101180]: pgmap v1856: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:52.512 INFO:teuthology.orchestra.run.smithi001.stdout:1597727834255 2024-09-14T04:35:53.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:52 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/4175502158' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:35:53.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:52 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/4175502158' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:35:53.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:52 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/4175502158' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:35:54.053 INFO:teuthology.orchestra.run.smithi001.stdout:133143986854 2024-09-14T04:35:54.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:53 smithi001 ceph-mon[107399]: pgmap v1857: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:54.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:53 smithi001 ceph-mon[112165]: pgmap v1857: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:54.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:53 smithi012 ceph-mon[101180]: pgmap v1857: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:54.873 INFO:teuthology.orchestra.run.smithi001.stdout:558345748883 2024-09-14T04:35:55.129 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/970884579' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:35:55.129 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:54 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/3656110053' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:35:55.129 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/970884579' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:35:55.129 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:54 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/3656110053' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:35:55.151 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1597727834250 got 1597727834255 for osd.5 2024-09-14T04:35:55.151 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/970884579' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:35:55.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:54 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/3656110053' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:35:55.708 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986850 got 133143986854 for osd.4 2024-09-14T04:35:55.708 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:55.953 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:55 smithi001 ceph-mon[107399]: pgmap v1858: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:56.220 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:55 smithi001 ceph-mon[112165]: pgmap v1858: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:56.346 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:55 smithi012 ceph-mon[101180]: pgmap v1858: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:56.369 INFO:teuthology.orchestra.run.smithi001.stdout:1275605287104 2024-09-14T04:35:56.954 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:35:56 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:35:56] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.51.0" 2024-09-14T04:35:57.220 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:56 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1283512314' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:35:57.221 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:56 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1283512314' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:35:57.269 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:56 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1283512314' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:35:57.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1275605287100 got 1275605287104 for osd.2 2024-09-14T04:35:57.460 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:58.084 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:57 smithi001 ceph-mon[107399]: pgmap v1859: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:58.084 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:57 smithi001 ceph-mon[112165]: pgmap v1859: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:58.345 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:57 smithi012 ceph-mon[101180]: pgmap v1859: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:35:58.411 INFO:tasks.cephadm.ceph_manager.ceph:need seq 558345748877 got 558345748883 for osd.3 2024-09-14T04:35:58.411 DEBUG:teuthology.parallel:result is None 2024-09-14T04:35:59.117 INFO:teuthology.orchestra.run.smithi001.stdout:743029342542 2024-09-14T04:35:59.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:35:59 smithi001 ceph-mon[107399]: from='client.? 172.21.15.1:0/1084333403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:35:59.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:35:59 smithi001 ceph-mon[112165]: from='client.? 172.21.15.1:0/1084333403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:35:59.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:35:59 smithi012 ceph-mon[101180]: from='client.? 172.21.15.1:0/1084333403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:36:00.400 INFO:tasks.cephadm.ceph_manager.ceph:need seq 743029342537 got 743029342542 for osd.0 2024-09-14T04:36:00.400 DEBUG:teuthology.parallel:result is None 2024-09-14T04:36:00.400 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:36:00.400 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:36:00.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:00 smithi012 ceph-mon[101180]: pgmap v1860: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:00.720 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:00 smithi001 ceph-mon[107399]: pgmap v1860: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:00.721 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:00 smithi001 ceph-mon[112165]: pgmap v1860: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:01.179 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:36:02.470 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[112165]: pgmap v1861: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:02 smithi001 ceph-mon[107399]: pgmap v1861: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' 2024-09-14T04:36:02.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:02 smithi012 ceph-mon[101180]: pgmap v1861: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:03.299 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:36:03.300 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:36:04.043 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1861,"stamp":"2024-09-14T04:36:01.694128+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":23122,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4679,"ondisk_log_size":4679,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9178308,"kb_used_data":68580,"kb_used_omap":41,"kb_used_meta":9109590,"kb_avail":740782908,"statfs":{"total":767960285184,"available":758561697792,"internally_reserved":0,"allocated":70225920,"data_stored":40299223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42935,"internal_metadata":9328220233},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003238"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":983,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752277+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:35:13.752277+0000","last_peered":"2024-09-14T04:35:13.752277+0000","last_clean":"2024-09-14T04:35:13.752277+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:35:13.752277+0000","last_undegraded":"2024-09-14T04:35:13.752277+0000","last_fullsized":"2024-09-14T04:35:13.752277+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:33:13.896667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757333+0000","last_change":"2024-09-14T04:34:20.656491+0000","last_active":"2024-09-14T04:35:13.757333+0000","last_peered":"2024-09-14T04:35:13.757333+0000","last_clean":"2024-09-14T04:35:13.757333+0000","last_became_active":"2024-09-14T04:34:20.656169+0000","last_became_peered":"2024-09-14T04:34:20.656169+0000","last_unstale":"2024-09-14T04:35:13.757333+0000","last_undegraded":"2024-09-14T04:35:13.757333+0000","last_fullsized":"2024-09-14T04:35:13.757333+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:07:08.833533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":195,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737355+0000","last_change":"2024-09-14T04:33:56.345878+0000","last_active":"2024-09-14T04:35:12.737355+0000","last_peered":"2024-09-14T04:35:12.737355+0000","last_clean":"2024-09-14T04:35:12.737355+0000","last_became_active":"2024-09-14T04:33:56.334984+0000","last_became_peered":"2024-09-14T04:33:56.334984+0000","last_unstale":"2024-09-14T04:35:12.737355+0000","last_undegraded":"2024-09-14T04:35:12.737355+0000","last_fullsized":"2024-09-14T04:35:12.737355+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:26.707796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":859,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.764814+0000","last_change":"2024-09-14T04:35:13.763098+0000","last_active":"2024-09-14T04:35:13.764814+0000","last_peered":"2024-09-14T04:35:13.764814+0000","last_clean":"2024-09-14T04:35:13.764814+0000","last_became_active":"2024-09-14T04:35:13.762699+0000","last_became_peered":"2024-09-14T04:35:13.762699+0000","last_unstale":"2024-09-14T04:35:13.764814+0000","last_undegraded":"2024-09-14T04:35:13.764814+0000","last_fullsized":"2024-09-14T04:35:13.764814+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:16.784038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.762911+0000","last_change":"2024-09-14T04:35:13.762453+0000","last_active":"2024-09-14T04:35:13.762911+0000","last_peered":"2024-09-14T04:35:13.762911+0000","last_clean":"2024-09-14T04:35:13.762911+0000","last_became_active":"2024-09-14T04:35:13.762116+0000","last_became_peered":"2024-09-14T04:35:13.762116+0000","last_unstale":"2024-09-14T04:35:13.762911+0000","last_undegraded":"2024-09-14T04:35:13.762911+0000","last_fullsized":"2024-09-14T04:35:13.762911+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:59.591678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":803,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752221+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:35:13.752221+0000","last_peered":"2024-09-14T04:35:13.752221+0000","last_clean":"2024-09-14T04:35:13.752221+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:35:13.752221+0000","last_undegraded":"2024-09-14T04:35:13.752221+0000","last_fullsized":"2024-09-14T04:35:13.752221+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":965,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748125+0000","last_change":"2024-09-14T04:34:02.358526+0000","last_active":"2024-09-14T04:35:13.748125+0000","last_peered":"2024-09-14T04:35:13.748125+0000","last_clean":"2024-09-14T04:35:13.748125+0000","last_became_active":"2024-09-14T04:34:02.357982+0000","last_became_peered":"2024-09-14T04:34:02.357982+0000","last_unstale":"2024-09-14T04:35:13.748125+0000","last_undegraded":"2024-09-14T04:35:13.748125+0000","last_fullsized":"2024-09-14T04:35:13.748125+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:02.769602+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":789,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752971+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:35:13.752971+0000","last_peered":"2024-09-14T04:35:13.752971+0000","last_clean":"2024-09-14T04:35:13.752971+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:35:13.752971+0000","last_undegraded":"2024-09-14T04:35:13.752971+0000","last_fullsized":"2024-09-14T04:35:13.752971+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.736773+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:35:12.736773+0000","last_peered":"2024-09-14T04:35:12.736773+0000","last_clean":"2024-09-14T04:35:12.736773+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:35:12.736773+0000","last_undegraded":"2024-09-14T04:35:12.736773+0000","last_fullsized":"2024-09-14T04:35:12.736773+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":712,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.764734+0000","last_change":"2024-09-14T04:35:13.763616+0000","last_active":"2024-09-14T04:35:13.764734+0000","last_peered":"2024-09-14T04:35:13.764734+0000","last_clean":"2024-09-14T04:35:13.764734+0000","last_became_active":"2024-09-14T04:35:13.763083+0000","last_became_peered":"2024-09-14T04:35:13.763083+0000","last_unstale":"2024-09-14T04:35:13.764734+0000","last_undegraded":"2024-09-14T04:35:13.764734+0000","last_fullsized":"2024-09-14T04:35:13.764734+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:47:33.126366+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":397,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757681+0000","last_change":"2024-09-14T04:34:53.787766+0000","last_active":"2024-09-14T04:35:13.757681+0000","last_peered":"2024-09-14T04:35:13.757681+0000","last_clean":"2024-09-14T04:35:13.757681+0000","last_became_active":"2024-09-14T04:34:53.787249+0000","last_became_peered":"2024-09-14T04:34:53.787249+0000","last_unstale":"2024-09-14T04:35:13.757681+0000","last_undegraded":"2024-09-14T04:35:13.757681+0000","last_fullsized":"2024-09-14T04:35:13.757681+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:16.891079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":150,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.758978+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:35:13.758978+0000","last_peered":"2024-09-14T04:35:13.758978+0000","last_clean":"2024-09-14T04:35:13.758978+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:35:13.758978+0000","last_undegraded":"2024-09-14T04:35:13.758978+0000","last_fullsized":"2024-09-14T04:35:13.758978+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":921,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737783+0000","last_change":"2024-09-14T04:34:02.356617+0000","last_active":"2024-09-14T04:35:12.737783+0000","last_peered":"2024-09-14T04:35:12.737783+0000","last_clean":"2024-09-14T04:35:12.737783+0000","last_became_active":"2024-09-14T04:34:02.356019+0000","last_became_peered":"2024-09-14T04:34:02.356019+0000","last_unstale":"2024-09-14T04:35:12.737783+0000","last_undegraded":"2024-09-14T04:35:12.737783+0000","last_fullsized":"2024-09-14T04:35:12.737783+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:51.131145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:51.131145+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:18.362346+0000","scrub_duration":0.001864201,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":163,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.738277+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:35:12.738277+0000","last_peered":"2024-09-14T04:35:12.738277+0000","last_clean":"2024-09-14T04:35:12.738277+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:35:12.738277+0000","last_undegraded":"2024-09-14T04:35:12.738277+0000","last_fullsized":"2024-09-14T04:35:12.738277+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765402+0000","last_change":"2024-09-14T04:35:13.764216+0000","last_active":"2024-09-14T04:35:13.765402+0000","last_peered":"2024-09-14T04:35:13.765402+0000","last_clean":"2024-09-14T04:35:13.765402+0000","last_became_active":"2024-09-14T04:35:13.763708+0000","last_became_peered":"2024-09-14T04:35:13.763708+0000","last_unstale":"2024-09-14T04:35:13.765402+0000","last_undegraded":"2024-09-14T04:35:13.765402+0000","last_fullsized":"2024-09-14T04:35:13.765402+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:43.963537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1357,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757453+0000","last_change":"2024-09-14T04:33:49.259743+0000","last_active":"2024-09-14T04:35:13.757453+0000","last_peered":"2024-09-14T04:35:13.757453+0000","last_clean":"2024-09-14T04:35:13.757453+0000","last_became_active":"2024-09-14T04:33:48.758490+0000","last_became_peered":"2024-09-14T04:33:48.758490+0000","last_unstale":"2024-09-14T04:35:13.757453+0000","last_undegraded":"2024-09-14T04:35:13.757453+0000","last_fullsized":"2024-09-14T04:35:13.757453+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.259685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.259685+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:56.085704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":189,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.738178+0000","last_change":"2024-09-14T04:33:59.349119+0000","last_active":"2024-09-14T04:35:12.738178+0000","last_peered":"2024-09-14T04:35:12.738178+0000","last_clean":"2024-09-14T04:35:12.738178+0000","last_became_active":"2024-09-14T04:33:59.336323+0000","last_became_peered":"2024-09-14T04:33:59.336323+0000","last_unstale":"2024-09-14T04:35:12.738178+0000","last_undegraded":"2024-09-14T04:35:12.738178+0000","last_fullsized":"2024-09-14T04:35:12.738178+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:51.489919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":644,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744304+0000","last_change":"2024-09-14T04:33:59.335137+0000","last_active":"2024-09-14T04:35:12.744304+0000","last_peered":"2024-09-14T04:35:12.744304+0000","last_clean":"2024-09-14T04:35:12.744304+0000","last_became_active":"2024-09-14T04:33:59.334808+0000","last_became_peered":"2024-09-14T04:33:59.334808+0000","last_unstale":"2024-09-14T04:35:12.744304+0000","last_undegraded":"2024-09-14T04:35:12.744304+0000","last_fullsized":"2024-09-14T04:35:12.744304+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:33.958948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":539,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748009+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:35:12.748009+0000","last_peered":"2024-09-14T04:35:12.748009+0000","last_clean":"2024-09-14T04:35:12.748009+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:35:12.748009+0000","last_undegraded":"2024-09-14T04:35:12.748009+0000","last_fullsized":"2024-09-14T04:35:12.748009+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":176,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737970+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:35:12.737970+0000","last_peered":"2024-09-14T04:35:12.737970+0000","last_clean":"2024-09-14T04:35:12.737970+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:35:12.737970+0000","last_undegraded":"2024-09-14T04:35:12.737970+0000","last_fullsized":"2024-09-14T04:35:12.737970+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748054+0000","last_change":"2024-09-14T04:33:59.345940+0000","last_active":"2024-09-14T04:35:12.748054+0000","last_peered":"2024-09-14T04:35:12.748054+0000","last_clean":"2024-09-14T04:35:12.748054+0000","last_became_active":"2024-09-14T04:33:59.345690+0000","last_became_peered":"2024-09-14T04:33:59.345690+0000","last_unstale":"2024-09-14T04:35:12.748054+0000","last_undegraded":"2024-09-14T04:35:12.748054+0000","last_fullsized":"2024-09-14T04:35:12.748054+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:19.006679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":617,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747064+0000","last_change":"2024-09-14T04:33:49.523592+0000","last_active":"2024-09-14T04:35:12.747064+0000","last_peered":"2024-09-14T04:35:12.747064+0000","last_clean":"2024-09-14T04:35:12.747064+0000","last_became_active":"2024-09-14T04:33:48.759868+0000","last_became_peered":"2024-09-14T04:33:48.759868+0000","last_unstale":"2024-09-14T04:35:12.747064+0000","last_undegraded":"2024-09-14T04:35:12.747064+0000","last_fullsized":"2024-09-14T04:35:12.747064+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.523485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.523485+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:38.218380+0000","scrub_duration":0.00154894,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":462,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765600+0000","last_change":"2024-09-14T04:35:13.765040+0000","last_active":"2024-09-14T04:35:13.765600+0000","last_peered":"2024-09-14T04:35:13.765600+0000","last_clean":"2024-09-14T04:35:13.765600+0000","last_became_active":"2024-09-14T04:35:13.764527+0000","last_became_peered":"2024-09-14T04:35:13.764527+0000","last_unstale":"2024-09-14T04:35:13.765600+0000","last_undegraded":"2024-09-14T04:35:13.765600+0000","last_fullsized":"2024-09-14T04:35:13.765600+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:39.654987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":790,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753033+0000","last_change":"2024-09-14T04:33:59.341074+0000","last_active":"2024-09-14T04:35:13.753033+0000","last_peered":"2024-09-14T04:35:13.753033+0000","last_clean":"2024-09-14T04:35:13.753033+0000","last_became_active":"2024-09-14T04:33:59.340717+0000","last_became_peered":"2024-09-14T04:33:59.340717+0000","last_unstale":"2024-09-14T04:35:13.753033+0000","last_undegraded":"2024-09-14T04:35:13.753033+0000","last_fullsized":"2024-09-14T04:35:13.753033+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:22.803669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":981,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749278+0000","last_change":"2024-09-14T04:33:59.328955+0000","last_active":"2024-09-14T04:35:13.749278+0000","last_peered":"2024-09-14T04:35:13.749278+0000","last_clean":"2024-09-14T04:35:13.749278+0000","last_became_active":"2024-09-14T04:33:59.328562+0000","last_became_peered":"2024-09-14T04:33:59.328562+0000","last_unstale":"2024-09-14T04:35:13.749278+0000","last_undegraded":"2024-09-14T04:35:13.749278+0000","last_fullsized":"2024-09-14T04:35:13.749278+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.165396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.165396+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:06.475748+0000","scrub_duration":0.0017422690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":64,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757301+0000","last_change":"2024-09-14T04:34:20.509266+0000","last_active":"2024-09-14T04:35:13.757301+0000","last_peered":"2024-09-14T04:35:13.757301+0000","last_clean":"2024-09-14T04:35:13.757301+0000","last_became_active":"2024-09-14T04:34:20.508582+0000","last_became_peered":"2024-09-14T04:34:20.508582+0000","last_unstale":"2024-09-14T04:35:13.757301+0000","last_undegraded":"2024-09-14T04:35:13.757301+0000","last_fullsized":"2024-09-14T04:35:13.757301+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:36:32.195051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":628,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746848+0000","last_change":"2024-09-14T04:32:52.819885+0000","last_active":"2024-09-14T04:35:12.746848+0000","last_peered":"2024-09-14T04:35:12.746848+0000","last_clean":"2024-09-14T04:35:12.746848+0000","last_became_active":"2024-09-14T04:32:52.819034+0000","last_became_peered":"2024-09-14T04:32:52.819034+0000","last_unstale":"2024-09-14T04:35:12.746848+0000","last_undegraded":"2024-09-14T04:35:12.746848+0000","last_fullsized":"2024-09-14T04:35:12.746848+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:52.327969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":750,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:22.093913+0000","last_change":"2024-09-14T04:35:22.094000+0000","last_active":"2024-09-14T04:35:22.093913+0000","last_peered":"2024-09-14T04:35:22.093913+0000","last_clean":"2024-09-14T04:35:22.093913+0000","last_became_active":"2024-09-14T04:35:13.759137+0000","last_became_peered":"2024-09-14T04:35:13.759137+0000","last_unstale":"2024-09-14T04:35:22.093913+0000","last_undegraded":"2024-09-14T04:35:22.093913+0000","last_fullsized":"2024-09-14T04:35:22.093913+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:22.093897+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:22.093897+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:34.169748+0000","scrub_duration":0.0019979960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748164+0000","last_change":"2024-09-14T04:33:59.345684+0000","last_active":"2024-09-14T04:35:12.748164+0000","last_peered":"2024-09-14T04:35:12.748164+0000","last_clean":"2024-09-14T04:35:12.748164+0000","last_became_active":"2024-09-14T04:33:59.345345+0000","last_became_peered":"2024-09-14T04:33:59.345345+0000","last_unstale":"2024-09-14T04:35:12.748164+0000","last_undegraded":"2024-09-14T04:35:12.748164+0000","last_fullsized":"2024-09-14T04:35:12.748164+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:43:41.565785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":236,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747096+0000","last_change":"2024-09-14T04:33:59.341288+0000","last_active":"2024-09-14T04:35:12.747096+0000","last_peered":"2024-09-14T04:35:12.747096+0000","last_clean":"2024-09-14T04:35:12.747096+0000","last_became_active":"2024-09-14T04:33:59.340994+0000","last_became_peered":"2024-09-14T04:33:59.340994+0000","last_unstale":"2024-09-14T04:35:12.747096+0000","last_undegraded":"2024-09-14T04:35:12.747096+0000","last_fullsized":"2024-09-14T04:35:12.747096+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:05.725236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":1047,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747866+0000","last_change":"2024-09-14T04:34:02.358731+0000","last_active":"2024-09-14T04:35:13.747866+0000","last_peered":"2024-09-14T04:35:13.747866+0000","last_clean":"2024-09-14T04:35:13.747866+0000","last_became_active":"2024-09-14T04:34:02.357980+0000","last_became_peered":"2024-09-14T04:34:02.357980+0000","last_unstale":"2024-09-14T04:35:13.747866+0000","last_undegraded":"2024-09-14T04:35:13.747866+0000","last_fullsized":"2024-09-14T04:35:13.747866+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.113032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.113032+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:06.066978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":236,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744159+0000","last_change":"2024-09-14T04:33:56.336399+0000","last_active":"2024-09-14T04:35:12.744159+0000","last_peered":"2024-09-14T04:35:12.744159+0000","last_clean":"2024-09-14T04:35:12.744159+0000","last_became_active":"2024-09-14T04:33:56.336070+0000","last_became_peered":"2024-09-14T04:33:56.336070+0000","last_unstale":"2024-09-14T04:35:12.744159+0000","last_undegraded":"2024-09-14T04:35:12.744159+0000","last_fullsized":"2024-09-14T04:35:12.744159+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:08.324552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":634,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747258+0000","last_change":"2024-09-14T04:33:59.345358+0000","last_active":"2024-09-14T04:35:12.747258+0000","last_peered":"2024-09-14T04:35:12.747258+0000","last_clean":"2024-09-14T04:35:12.747258+0000","last_became_active":"2024-09-14T04:33:59.345038+0000","last_became_peered":"2024-09-14T04:33:59.345038+0000","last_unstale":"2024-09-14T04:35:12.747258+0000","last_undegraded":"2024-09-14T04:35:12.747258+0000","last_fullsized":"2024-09-14T04:35:12.747258+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:25.738959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":738,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:32.302067+0000","last_change":"2024-09-14T04:35:32.302153+0000","last_active":"2024-09-14T04:35:32.302067+0000","last_peered":"2024-09-14T04:35:32.302067+0000","last_clean":"2024-09-14T04:35:32.302067+0000","last_became_active":"2024-09-14T04:35:13.763207+0000","last_became_peered":"2024-09-14T04:35:13.763207+0000","last_unstale":"2024-09-14T04:35:32.302067+0000","last_undegraded":"2024-09-14T04:35:32.302067+0000","last_fullsized":"2024-09-14T04:35:32.302067+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:32.302048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:32.302048+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:10.908568+0000","scrub_duration":0.001810346,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":383,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756961+0000","last_change":"2024-09-14T04:33:59.340304+0000","last_active":"2024-09-14T04:35:13.756961+0000","last_peered":"2024-09-14T04:35:13.756961+0000","last_clean":"2024-09-14T04:35:13.756961+0000","last_became_active":"2024-09-14T04:33:59.339996+0000","last_became_peered":"2024-09-14T04:33:59.339996+0000","last_unstale":"2024-09-14T04:35:13.756961+0000","last_undegraded":"2024-09-14T04:35:13.756961+0000","last_fullsized":"2024-09-14T04:35:13.756961+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:11.254308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748038+0000","last_change":"2024-09-14T04:33:59.334638+0000","last_active":"2024-09-14T04:35:13.748038+0000","last_peered":"2024-09-14T04:35:13.748038+0000","last_clean":"2024-09-14T04:35:13.748038+0000","last_became_active":"2024-09-14T04:33:59.334323+0000","last_became_peered":"2024-09-14T04:33:59.334323+0000","last_unstale":"2024-09-14T04:35:13.748038+0000","last_undegraded":"2024-09-14T04:35:13.748038+0000","last_fullsized":"2024-09-14T04:35:13.748038+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:20.689612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":998,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749486+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:35:13.749486+0000","last_peered":"2024-09-14T04:35:13.749486+0000","last_clean":"2024-09-14T04:35:13.749486+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:35:13.749486+0000","last_undegraded":"2024-09-14T04:35:13.749486+0000","last_fullsized":"2024-09-14T04:35:13.749486+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":456,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.762507+0000","last_change":"2024-09-14T04:35:13.761860+0000","last_active":"2024-09-14T04:35:13.762507+0000","last_peered":"2024-09-14T04:35:13.762507+0000","last_clean":"2024-09-14T04:35:13.762507+0000","last_became_active":"2024-09-14T04:35:13.761215+0000","last_became_peered":"2024-09-14T04:35:13.761215+0000","last_unstale":"2024-09-14T04:35:13.762507+0000","last_undegraded":"2024-09-14T04:35:13.762507+0000","last_fullsized":"2024-09-14T04:35:13.762507+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:07.554220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757082+0000","last_change":"2024-09-14T04:34:20.507274+0000","last_active":"2024-09-14T04:35:13.757082+0000","last_peered":"2024-09-14T04:35:13.757082+0000","last_clean":"2024-09-14T04:35:13.757082+0000","last_became_active":"2024-09-14T04:34:20.506964+0000","last_became_peered":"2024-09-14T04:34:20.506964+0000","last_unstale":"2024-09-14T04:35:13.757082+0000","last_undegraded":"2024-09-14T04:35:13.757082+0000","last_fullsized":"2024-09-14T04:35:13.757082+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:26:57.758549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746320+0000","last_change":"2024-09-14T04:32:52.817519+0000","last_active":"2024-09-14T04:35:12.746320+0000","last_peered":"2024-09-14T04:35:12.746320+0000","last_clean":"2024-09-14T04:35:12.746320+0000","last_became_active":"2024-09-14T04:32:52.817207+0000","last_became_peered":"2024-09-14T04:32:52.817207+0000","last_unstale":"2024-09-14T04:35:12.746320+0000","last_undegraded":"2024-09-14T04:35:12.746320+0000","last_fullsized":"2024-09-14T04:35:12.746320+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:43.967076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":468,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747943+0000","last_change":"2024-09-14T04:34:03.164615+0000","last_active":"2024-09-14T04:35:13.747943+0000","last_peered":"2024-09-14T04:35:13.747943+0000","last_clean":"2024-09-14T04:35:13.747943+0000","last_became_active":"2024-09-14T04:34:02.355231+0000","last_became_peered":"2024-09-14T04:34:02.355231+0000","last_unstale":"2024-09-14T04:35:13.747943+0000","last_undegraded":"2024-09-14T04:35:13.747943+0000","last_fullsized":"2024-09-14T04:35:13.747943+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:03.164560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:03.164560+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:38:27.410822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":178,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749382+0000","last_change":"2024-09-14T04:33:59.334578+0000","last_active":"2024-09-14T04:35:13.749382+0000","last_peered":"2024-09-14T04:35:13.749382+0000","last_clean":"2024-09-14T04:35:13.749382+0000","last_became_active":"2024-09-14T04:33:59.334246+0000","last_became_peered":"2024-09-14T04:33:59.334246+0000","last_unstale":"2024-09-14T04:35:13.749382+0000","last_undegraded":"2024-09-14T04:35:13.749382+0000","last_fullsized":"2024-09-14T04:35:13.749382+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:01.401424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1026,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752279+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:35:13.752279+0000","last_peered":"2024-09-14T04:35:13.752279+0000","last_clean":"2024-09-14T04:35:13.752279+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:35:13.752279+0000","last_undegraded":"2024-09-14T04:35:13.752279+0000","last_fullsized":"2024-09-14T04:35:13.752279+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":815,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753056+0000","last_change":"2024-09-14T04:33:59.335406+0000","last_active":"2024-09-14T04:35:13.753056+0000","last_peered":"2024-09-14T04:35:13.753056+0000","last_clean":"2024-09-14T04:35:13.753056+0000","last_became_active":"2024-09-14T04:33:59.334973+0000","last_became_peered":"2024-09-14T04:33:59.334973+0000","last_unstale":"2024-09-14T04:35:13.753056+0000","last_undegraded":"2024-09-14T04:35:13.753056+0000","last_fullsized":"2024-09-14T04:35:13.753056+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:28.459636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":856,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749342+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:35:13.749342+0000","last_peered":"2024-09-14T04:35:13.749342+0000","last_clean":"2024-09-14T04:35:13.749342+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:35:13.749342+0000","last_undegraded":"2024-09-14T04:35:13.749342+0000","last_fullsized":"2024-09-14T04:35:13.749342+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":62,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753884+0000","last_change":"2024-09-14T04:33:56.335336+0000","last_active":"2024-09-14T04:35:13.753884+0000","last_peered":"2024-09-14T04:35:13.753884+0000","last_clean":"2024-09-14T04:35:13.753884+0000","last_became_active":"2024-09-14T04:33:56.335029+0000","last_became_peered":"2024-09-14T04:33:56.335029+0000","last_unstale":"2024-09-14T04:35:13.753884+0000","last_undegraded":"2024-09-14T04:35:13.753884+0000","last_fullsized":"2024-09-14T04:35:13.753884+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:31.715946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747704+0000","last_change":"2024-09-14T04:33:59.339238+0000","last_active":"2024-09-14T04:35:13.747704+0000","last_peered":"2024-09-14T04:35:13.747704+0000","last_clean":"2024-09-14T04:35:13.747704+0000","last_became_active":"2024-09-14T04:33:59.338916+0000","last_became_peered":"2024-09-14T04:33:59.338916+0000","last_unstale":"2024-09-14T04:35:13.747704+0000","last_undegraded":"2024-09-14T04:35:13.747704+0000","last_fullsized":"2024-09-14T04:35:13.747704+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:00.746903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747923+0000","last_change":"2024-09-14T04:33:59.333665+0000","last_active":"2024-09-14T04:35:13.747923+0000","last_peered":"2024-09-14T04:35:13.747923+0000","last_clean":"2024-09-14T04:35:13.747923+0000","last_became_active":"2024-09-14T04:33:59.331169+0000","last_became_peered":"2024-09-14T04:33:59.331169+0000","last_unstale":"2024-09-14T04:35:13.747923+0000","last_undegraded":"2024-09-14T04:35:13.747923+0000","last_fullsized":"2024-09-14T04:35:13.747923+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:53:01.798897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":732,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.759398+0000","last_change":"2024-09-14T04:35:13.759000+0000","last_active":"2024-09-14T04:35:13.759398+0000","last_peered":"2024-09-14T04:35:13.759398+0000","last_clean":"2024-09-14T04:35:13.759398+0000","last_became_active":"2024-09-14T04:35:13.758415+0000","last_became_peered":"2024-09-14T04:35:13.758415+0000","last_unstale":"2024-09-14T04:35:13.759398+0000","last_undegraded":"2024-09-14T04:35:13.759398+0000","last_fullsized":"2024-09-14T04:35:13.759398+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:54:04.352518+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":440,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765465+0000","last_change":"2024-09-14T04:35:13.764189+0000","last_active":"2024-09-14T04:35:13.765465+0000","last_peered":"2024-09-14T04:35:13.765465+0000","last_clean":"2024-09-14T04:35:13.765465+0000","last_became_active":"2024-09-14T04:35:13.763794+0000","last_became_peered":"2024-09-14T04:35:13.763794+0000","last_unstale":"2024-09-14T04:35:13.765465+0000","last_undegraded":"2024-09-14T04:35:13.765465+0000","last_fullsized":"2024-09-14T04:35:13.765465+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:34.136506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":619,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746545+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:35:12.746545+0000","last_peered":"2024-09-14T04:35:12.746545+0000","last_clean":"2024-09-14T04:35:12.746545+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:35:12.746545+0000","last_undegraded":"2024-09-14T04:35:12.746545+0000","last_fullsized":"2024-09-14T04:35:12.746545+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748074+0000","last_change":"2024-09-14T04:33:59.333265+0000","last_active":"2024-09-14T04:35:13.748074+0000","last_peered":"2024-09-14T04:35:13.748074+0000","last_clean":"2024-09-14T04:35:13.748074+0000","last_became_active":"2024-09-14T04:33:59.332915+0000","last_became_peered":"2024-09-14T04:33:59.332915+0000","last_unstale":"2024-09-14T04:35:13.748074+0000","last_undegraded":"2024-09-14T04:35:13.748074+0000","last_fullsized":"2024-09-14T04:35:13.748074+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:58:07.942027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":1004,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747816+0000","last_change":"2024-09-14T04:33:49.324876+0000","last_active":"2024-09-14T04:35:12.747816+0000","last_peered":"2024-09-14T04:35:12.747816+0000","last_clean":"2024-09-14T04:35:12.747816+0000","last_became_active":"2024-09-14T04:33:48.761757+0000","last_became_peered":"2024-09-14T04:33:48.761757+0000","last_unstale":"2024-09-14T04:35:12.747816+0000","last_undegraded":"2024-09-14T04:35:12.747816+0000","last_fullsized":"2024-09-14T04:35:12.747816+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.324749+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.324749+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:30.576200+0000","scrub_duration":0.001637119,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":602,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.759450+0000","last_change":"2024-09-14T04:35:13.758908+0000","last_active":"2024-09-14T04:35:13.759450+0000","last_peered":"2024-09-14T04:35:13.759450+0000","last_clean":"2024-09-14T04:35:13.759450+0000","last_became_active":"2024-09-14T04:35:13.758337+0000","last_became_peered":"2024-09-14T04:35:13.758337+0000","last_unstale":"2024-09-14T04:35:13.759450+0000","last_undegraded":"2024-09-14T04:35:13.759450+0000","last_fullsized":"2024-09-14T04:35:13.759450+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:29:38.259259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748222+0000","last_change":"2024-09-14T04:33:59.334159+0000","last_active":"2024-09-14T04:35:13.748222+0000","last_peered":"2024-09-14T04:35:13.748222+0000","last_clean":"2024-09-14T04:35:13.748222+0000","last_became_active":"2024-09-14T04:33:59.333204+0000","last_became_peered":"2024-09-14T04:33:59.333204+0000","last_unstale":"2024-09-14T04:35:13.748222+0000","last_undegraded":"2024-09-14T04:35:13.748222+0000","last_fullsized":"2024-09-14T04:35:13.748222+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:39:16.678016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748066+0000","last_change":"2024-09-14T04:33:59.335699+0000","last_active":"2024-09-14T04:35:13.748066+0000","last_peered":"2024-09-14T04:35:13.748066+0000","last_clean":"2024-09-14T04:35:13.748066+0000","last_became_active":"2024-09-14T04:33:59.335204+0000","last_became_peered":"2024-09-14T04:33:59.335204+0000","last_unstale":"2024-09-14T04:35:13.748066+0000","last_undegraded":"2024-09-14T04:35:13.748066+0000","last_fullsized":"2024-09-14T04:35:13.748066+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:14.368649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":934,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748323+0000","last_change":"2024-09-14T04:33:50.279734+0000","last_active":"2024-09-14T04:35:12.748323+0000","last_peered":"2024-09-14T04:35:12.748323+0000","last_clean":"2024-09-14T04:35:12.748323+0000","last_became_active":"2024-09-14T04:33:48.761487+0000","last_became_peered":"2024-09-14T04:33:48.761487+0000","last_unstale":"2024-09-14T04:35:12.748323+0000","last_undegraded":"2024-09-14T04:35:12.748323+0000","last_fullsized":"2024-09-14T04:35:12.748323+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:50.279615+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:50.279615+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:37:04.686428+0000","scrub_duration":0.001448866,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748320+0000","last_change":"2024-09-14T04:33:59.335857+0000","last_active":"2024-09-14T04:35:13.748320+0000","last_peered":"2024-09-14T04:35:13.748320+0000","last_clean":"2024-09-14T04:35:13.748320+0000","last_became_active":"2024-09-14T04:33:59.334539+0000","last_became_peered":"2024-09-14T04:33:59.334539+0000","last_unstale":"2024-09-14T04:35:13.748320+0000","last_undegraded":"2024-09-14T04:35:13.748320+0000","last_fullsized":"2024-09-14T04:35:13.748320+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:53.748882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":160,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749937+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:35:13.749937+0000","last_peered":"2024-09-14T04:35:13.749937+0000","last_clean":"2024-09-14T04:35:13.749937+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:35:13.749937+0000","last_undegraded":"2024-09-14T04:35:13.749937+0000","last_fullsized":"2024-09-14T04:35:13.749937+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":800,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752942+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:35:13.752942+0000","last_peered":"2024-09-14T04:35:13.752942+0000","last_clean":"2024-09-14T04:35:13.752942+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:35:13.752942+0000","last_undegraded":"2024-09-14T04:35:13.752942+0000","last_fullsized":"2024-09-14T04:35:13.752942+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":969,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756095+0000","last_change":"2024-09-14T04:33:59.340118+0000","last_active":"2024-09-14T04:35:13.756095+0000","last_peered":"2024-09-14T04:35:13.756095+0000","last_clean":"2024-09-14T04:35:13.756095+0000","last_became_active":"2024-09-14T04:33:59.339690+0000","last_became_peered":"2024-09-14T04:33:59.339690+0000","last_unstale":"2024-09-14T04:35:13.756095+0000","last_undegraded":"2024-09-14T04:35:13.756095+0000","last_fullsized":"2024-09-14T04:35:13.756095+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:52.546611+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:33:52.546611+0000","last_clean_scrub_stamp":"2024-09-14T04:33:52.546611+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:53.371259+0000","scrub_duration":0.001481095,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":64,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757781+0000","last_change":"2024-09-14T04:34:20.508620+0000","last_active":"2024-09-14T04:35:13.757781+0000","last_peered":"2024-09-14T04:35:13.757781+0000","last_clean":"2024-09-14T04:35:13.757781+0000","last_became_active":"2024-09-14T04:34:20.508315+0000","last_became_peered":"2024-09-14T04:34:20.508315+0000","last_unstale":"2024-09-14T04:35:13.757781+0000","last_undegraded":"2024-09-14T04:35:13.757781+0000","last_fullsized":"2024-09-14T04:35:13.757781+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:05:19.179517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748291+0000","last_change":"2024-09-14T04:33:59.345511+0000","last_active":"2024-09-14T04:35:13.748291+0000","last_peered":"2024-09-14T04:35:13.748291+0000","last_clean":"2024-09-14T04:35:13.748291+0000","last_became_active":"2024-09-14T04:33:59.345081+0000","last_became_peered":"2024-09-14T04:33:59.345081+0000","last_unstale":"2024-09-14T04:35:13.748291+0000","last_undegraded":"2024-09-14T04:35:13.748291+0000","last_fullsized":"2024-09-14T04:35:13.748291+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:14:14.641270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":393,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756852+0000","last_change":"2024-09-14T04:34:20.508544+0000","last_active":"2024-09-14T04:35:13.756852+0000","last_peered":"2024-09-14T04:35:13.756852+0000","last_clean":"2024-09-14T04:35:13.756852+0000","last_became_active":"2024-09-14T04:34:20.508251+0000","last_became_peered":"2024-09-14T04:34:20.508251+0000","last_unstale":"2024-09-14T04:35:13.756852+0000","last_undegraded":"2024-09-14T04:35:13.756852+0000","last_fullsized":"2024-09-14T04:35:13.756852+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:38.540818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1318,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757487+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:35:13.757487+0000","last_peered":"2024-09-14T04:35:13.757487+0000","last_clean":"2024-09-14T04:35:13.757487+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:35:13.757487+0000","last_undegraded":"2024-09-14T04:35:13.757487+0000","last_fullsized":"2024-09-14T04:35:13.757487+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:15.915497+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":813,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752946+0000","last_change":"2024-09-14T04:33:59.346006+0000","last_active":"2024-09-14T04:35:13.752946+0000","last_peered":"2024-09-14T04:35:13.752946+0000","last_clean":"2024-09-14T04:35:13.752946+0000","last_became_active":"2024-09-14T04:33:59.336636+0000","last_became_peered":"2024-09-14T04:33:59.336636+0000","last_unstale":"2024-09-14T04:35:13.752946+0000","last_undegraded":"2024-09-14T04:35:13.752946+0000","last_fullsized":"2024-09-14T04:35:13.752946+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:42:49.996255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747914+0000","last_change":"2024-09-14T04:33:56.343398+0000","last_active":"2024-09-14T04:35:12.747914+0000","last_peered":"2024-09-14T04:35:12.747914+0000","last_clean":"2024-09-14T04:35:12.747914+0000","last_became_active":"2024-09-14T04:33:56.343047+0000","last_became_peered":"2024-09-14T04:33:56.343047+0000","last_unstale":"2024-09-14T04:35:12.747914+0000","last_undegraded":"2024-09-14T04:35:12.747914+0000","last_fullsized":"2024-09-14T04:35:12.747914+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:50.690046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":585,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744375+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:35:12.744375+0000","last_peered":"2024-09-14T04:35:12.744375+0000","last_clean":"2024-09-14T04:35:12.744375+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:35:12.744375+0000","last_undegraded":"2024-09-14T04:35:12.744375+0000","last_fullsized":"2024-09-14T04:35:12.744375+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"86'149","reported_seq":862,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748336+0000","last_change":"2024-09-14T04:34:04.164795+0000","last_active":"2024-09-14T04:35:13.748336+0000","last_peered":"2024-09-14T04:35:13.748336+0000","last_clean":"2024-09-14T04:35:13.748336+0000","last_became_active":"2024-09-14T04:34:02.356941+0000","last_became_peered":"2024-09-14T04:34:02.356941+0000","last_unstale":"2024-09-14T04:35:13.748336+0000","last_undegraded":"2024-09-14T04:35:13.748336+0000","last_fullsized":"2024-09-14T04:35:13.748336+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:04.164744+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:04.164744+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:31:14.475505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":648,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746599+0000","last_change":"2024-09-14T04:33:59.341470+0000","last_active":"2024-09-14T04:35:12.746599+0000","last_peered":"2024-09-14T04:35:12.746599+0000","last_clean":"2024-09-14T04:35:12.746599+0000","last_became_active":"2024-09-14T04:33:59.341164+0000","last_became_peered":"2024-09-14T04:33:59.341164+0000","last_unstale":"2024-09-14T04:35:12.746599+0000","last_undegraded":"2024-09-14T04:35:12.746599+0000","last_fullsized":"2024-09-14T04:35:12.746599+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:07.220907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":649,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747316+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:35:12.747316+0000","last_peered":"2024-09-14T04:35:12.747316+0000","last_clean":"2024-09-14T04:35:12.747316+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:35:12.747316+0000","last_undegraded":"2024-09-14T04:35:12.747316+0000","last_fullsized":"2024-09-14T04:35:12.747316+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"86'149","reported_seq":956,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:14.042036+0000","last_change":"2024-09-14T04:35:14.042036+0000","last_active":"2024-09-14T04:35:14.042036+0000","last_peered":"2024-09-14T04:35:14.042036+0000","last_clean":"2024-09-14T04:35:14.042036+0000","last_became_active":"2024-09-14T04:35:13.763481+0000","last_became_peered":"2024-09-14T04:35:13.763481+0000","last_unstale":"2024-09-14T04:35:14.042036+0000","last_undegraded":"2024-09-14T04:35:14.042036+0000","last_fullsized":"2024-09-14T04:35:14.042036+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:14.041928+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:35:14.041928+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:51.392429+0000","scrub_duration":0.0016617579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747852+0000","last_change":"2024-09-14T04:33:59.340411+0000","last_active":"2024-09-14T04:35:13.747852+0000","last_peered":"2024-09-14T04:35:13.747852+0000","last_clean":"2024-09-14T04:35:13.747852+0000","last_became_active":"2024-09-14T04:33:59.340172+0000","last_became_peered":"2024-09-14T04:33:59.340172+0000","last_unstale":"2024-09-14T04:35:13.747852+0000","last_undegraded":"2024-09-14T04:35:13.747852+0000","last_fullsized":"2024-09-14T04:35:13.747852+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:23:06.457371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":165,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737386+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:35:12.737386+0000","last_peered":"2024-09-14T04:35:12.737386+0000","last_clean":"2024-09-14T04:35:12.737386+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:35:12.737386+0000","last_undegraded":"2024-09-14T04:35:12.737386+0000","last_fullsized":"2024-09-14T04:35:12.737386+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":913,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753770+0000","last_change":"2024-09-14T04:33:59.331662+0000","last_active":"2024-09-14T04:35:13.753770+0000","last_peered":"2024-09-14T04:35:13.753770+0000","last_clean":"2024-09-14T04:35:13.753770+0000","last_became_active":"2024-09-14T04:33:59.331251+0000","last_became_peered":"2024-09-14T04:33:59.331251+0000","last_unstale":"2024-09-14T04:35:13.753770+0000","last_undegraded":"2024-09-14T04:35:13.753770+0000","last_fullsized":"2024-09-14T04:35:13.753770+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:43.956655+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":439,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.758013+0000","last_change":"2024-09-14T04:35:13.756946+0000","last_active":"2024-09-14T04:35:13.758013+0000","last_peered":"2024-09-14T04:35:13.758013+0000","last_clean":"2024-09-14T04:35:13.758013+0000","last_became_active":"2024-09-14T04:35:13.756615+0000","last_became_peered":"2024-09-14T04:35:13.756615+0000","last_unstale":"2024-09-14T04:35:13.758013+0000","last_undegraded":"2024-09-14T04:35:13.758013+0000","last_fullsized":"2024-09-14T04:35:13.758013+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:25.517599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757291+0000","last_change":"2024-09-14T04:34:20.506624+0000","last_active":"2024-09-14T04:35:13.757291+0000","last_peered":"2024-09-14T04:35:13.757291+0000","last_clean":"2024-09-14T04:35:13.757291+0000","last_became_active":"2024-09-14T04:34:20.506213+0000","last_became_peered":"2024-09-14T04:34:20.506213+0000","last_unstale":"2024-09-14T04:35:13.757291+0000","last_undegraded":"2024-09-14T04:35:13.757291+0000","last_fullsized":"2024-09-14T04:35:13.757291+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:16.454233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"86'149","reported_seq":977,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737550+0000","last_change":"2024-09-14T04:34:03.233993+0000","last_active":"2024-09-14T04:35:12.737550+0000","last_peered":"2024-09-14T04:35:12.737550+0000","last_clean":"2024-09-14T04:35:12.737550+0000","last_became_active":"2024-09-14T04:34:02.355842+0000","last_became_peered":"2024-09-14T04:34:02.355842+0000","last_unstale":"2024-09-14T04:35:12.737550+0000","last_undegraded":"2024-09-14T04:35:12.737550+0000","last_fullsized":"2024-09-14T04:35:12.737550+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:03.233909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:03.233909+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:38.397295+0000","scrub_duration":0.0021438249999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":786,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753209+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:35:13.753209+0000","last_peered":"2024-09-14T04:35:13.753209+0000","last_clean":"2024-09-14T04:35:13.753209+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:35:13.753209+0000","last_undegraded":"2024-09-14T04:35:13.753209+0000","last_fullsized":"2024-09-14T04:35:13.753209+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":912,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757073+0000","last_change":"2024-09-14T04:33:59.339494+0000","last_active":"2024-09-14T04:35:13.757073+0000","last_peered":"2024-09-14T04:35:13.757073+0000","last_clean":"2024-09-14T04:35:13.757073+0000","last_became_active":"2024-09-14T04:33:59.339188+0000","last_became_peered":"2024-09-14T04:33:59.339188+0000","last_unstale":"2024-09-14T04:35:13.757073+0000","last_undegraded":"2024-09-14T04:35:13.757073+0000","last_fullsized":"2024-09-14T04:35:13.757073+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:38:20.215115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"86'149","reported_seq":748,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744545+0000","last_change":"2024-09-14T04:34:08.551968+0000","last_active":"2024-09-14T04:35:12.744545+0000","last_peered":"2024-09-14T04:35:12.744545+0000","last_clean":"2024-09-14T04:35:12.744545+0000","last_became_active":"2024-09-14T04:34:02.358643+0000","last_became_peered":"2024-09-14T04:34:02.358643+0000","last_unstale":"2024-09-14T04:35:12.744545+0000","last_undegraded":"2024-09-14T04:35:12.744545+0000","last_fullsized":"2024-09-14T04:35:12.744545+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:08.551862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:08.551862+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:34.530388+0000","scrub_duration":0.001558928,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748098+0000","last_change":"2024-09-14T04:33:59.339591+0000","last_active":"2024-09-14T04:35:12.748098+0000","last_peered":"2024-09-14T04:35:12.748098+0000","last_clean":"2024-09-14T04:35:12.748098+0000","last_became_active":"2024-09-14T04:33:59.339122+0000","last_became_peered":"2024-09-14T04:33:59.339122+0000","last_unstale":"2024-09-14T04:35:12.748098+0000","last_undegraded":"2024-09-14T04:35:12.748098+0000","last_fullsized":"2024-09-14T04:35:12.748098+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:52.809466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":351,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757216+0000","last_change":"2024-09-14T04:34:20.656480+0000","last_active":"2024-09-14T04:35:13.757216+0000","last_peered":"2024-09-14T04:35:13.757216+0000","last_clean":"2024-09-14T04:35:13.757216+0000","last_became_active":"2024-09-14T04:34:20.656118+0000","last_became_peered":"2024-09-14T04:34:20.656118+0000","last_unstale":"2024-09-14T04:35:13.757216+0000","last_undegraded":"2024-09-14T04:35:13.757216+0000","last_fullsized":"2024-09-14T04:35:13.757216+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:29.581369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"86'149","reported_seq":546,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:25.176595+0000","last_change":"2024-09-14T04:35:25.176728+0000","last_active":"2024-09-14T04:35:25.176595+0000","last_peered":"2024-09-14T04:35:25.176595+0000","last_clean":"2024-09-14T04:35:25.176595+0000","last_became_active":"2024-09-14T04:35:13.760243+0000","last_became_peered":"2024-09-14T04:35:13.760243+0000","last_unstale":"2024-09-14T04:35:25.176595+0000","last_undegraded":"2024-09-14T04:35:25.176595+0000","last_fullsized":"2024-09-14T04:35:25.176595+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:25.176581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:25.176581+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:46.706478+0000","scrub_duration":0.002187488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":810,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752858+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:35:13.752858+0000","last_peered":"2024-09-14T04:35:13.752858+0000","last_clean":"2024-09-14T04:35:13.752858+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:35:13.752858+0000","last_undegraded":"2024-09-14T04:35:13.752858+0000","last_fullsized":"2024-09-14T04:35:13.752858+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747675+0000","last_change":"2024-09-14T04:33:59.343158+0000","last_active":"2024-09-14T04:35:13.747675+0000","last_peered":"2024-09-14T04:35:13.747675+0000","last_clean":"2024-09-14T04:35:13.747675+0000","last_became_active":"2024-09-14T04:33:59.342282+0000","last_became_peered":"2024-09-14T04:33:59.342282+0000","last_unstale":"2024-09-14T04:35:13.747675+0000","last_undegraded":"2024-09-14T04:35:13.747675+0000","last_fullsized":"2024-09-14T04:35:13.747675+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:50:05.754340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"587'150","reported_seq":751,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:29.251778+0000","last_change":"2024-09-14T04:35:29.251897+0000","last_active":"2024-09-14T04:35:29.251778+0000","last_peered":"2024-09-14T04:35:29.251778+0000","last_clean":"2024-09-14T04:35:29.251778+0000","last_became_active":"2024-09-14T04:35:13.761178+0000","last_became_peered":"2024-09-14T04:35:13.761178+0000","last_unstale":"2024-09-14T04:35:29.251778+0000","last_undegraded":"2024-09-14T04:35:29.251778+0000","last_fullsized":"2024-09-14T04:35:29.251778+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"587'150","last_scrub_stamp":"2024-09-14T04:35:29.251757+0000","last_deep_scrub":"587'150","last_deep_scrub_stamp":"2024-09-14T04:35:29.251757+0000","last_clean_scrub_stamp":"2024-09-14T04:35:29.251757+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:30.874703+0000","scrub_duration":0.0035824609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":319,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757523+0000","last_change":"2024-09-14T04:34:54.389583+0000","last_active":"2024-09-14T04:35:13.757523+0000","last_peered":"2024-09-14T04:35:13.757523+0000","last_clean":"2024-09-14T04:35:13.757523+0000","last_became_active":"2024-09-14T04:34:54.389299+0000","last_became_peered":"2024-09-14T04:34:54.389299+0000","last_unstale":"2024-09-14T04:35:13.757523+0000","last_undegraded":"2024-09-14T04:35:13.757523+0000","last_fullsized":"2024-09-14T04:35:13.757523+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:03.599428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":216,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754812+0000","last_change":"2024-09-14T04:33:59.346158+0000","last_active":"2024-09-14T04:35:13.754812+0000","last_peered":"2024-09-14T04:35:13.754812+0000","last_clean":"2024-09-14T04:35:13.754812+0000","last_became_active":"2024-09-14T04:33:59.345759+0000","last_became_peered":"2024-09-14T04:33:59.345759+0000","last_unstale":"2024-09-14T04:35:13.754812+0000","last_undegraded":"2024-09-14T04:35:13.754812+0000","last_fullsized":"2024-09-14T04:35:13.754812+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:51:25.669328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"86'149","reported_seq":559,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756948+0000","last_change":"2024-09-14T04:34:21.324557+0000","last_active":"2024-09-14T04:35:13.756948+0000","last_peered":"2024-09-14T04:35:13.756948+0000","last_clean":"2024-09-14T04:35:13.756948+0000","last_became_active":"2024-09-14T04:34:20.509879+0000","last_became_peered":"2024-09-14T04:34:20.509879+0000","last_unstale":"2024-09-14T04:35:13.756948+0000","last_undegraded":"2024-09-14T04:35:13.756948+0000","last_fullsized":"2024-09-14T04:35:13.756948+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:21.324506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:21.324506+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:33.643045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":76,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754007+0000","last_change":"2024-09-14T04:34:53.789706+0000","last_active":"2024-09-14T04:35:13.754007+0000","last_peered":"2024-09-14T04:35:13.754007+0000","last_clean":"2024-09-14T04:35:13.754007+0000","last_became_active":"2024-09-14T04:34:53.789424+0000","last_became_peered":"2024-09-14T04:34:53.789424+0000","last_unstale":"2024-09-14T04:35:13.754007+0000","last_undegraded":"2024-09-14T04:35:13.754007+0000","last_fullsized":"2024-09-14T04:35:13.754007+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:49.777958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":814,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752787+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:35:13.752787+0000","last_peered":"2024-09-14T04:35:13.752787+0000","last_clean":"2024-09-14T04:35:13.752787+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:35:13.752787+0000","last_undegraded":"2024-09-14T04:35:13.752787+0000","last_fullsized":"2024-09-14T04:35:13.752787+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"86'149","reported_seq":481,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757745+0000","last_change":"2024-09-14T04:33:58.052578+0000","last_active":"2024-09-14T04:35:13.757745+0000","last_peered":"2024-09-14T04:35:13.757745+0000","last_clean":"2024-09-14T04:35:13.757745+0000","last_became_active":"2024-09-14T04:33:57.336734+0000","last_became_peered":"2024-09-14T04:33:57.336734+0000","last_unstale":"2024-09-14T04:35:13.757745+0000","last_undegraded":"2024-09-14T04:35:13.757745+0000","last_fullsized":"2024-09-14T04:35:13.757745+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:58.052531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:58.052531+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:11.321631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748699+0000","last_change":"2024-09-14T04:33:59.336010+0000","last_active":"2024-09-14T04:35:12.748699+0000","last_peered":"2024-09-14T04:35:12.748699+0000","last_clean":"2024-09-14T04:35:12.748699+0000","last_became_active":"2024-09-14T04:33:59.335702+0000","last_became_peered":"2024-09-14T04:33:59.335702+0000","last_unstale":"2024-09-14T04:35:12.748699+0000","last_undegraded":"2024-09-14T04:35:12.748699+0000","last_fullsized":"2024-09-14T04:35:12.748699+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:57.635601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":452,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.761300+0000","last_change":"2024-09-14T04:35:13.760793+0000","last_active":"2024-09-14T04:35:13.761300+0000","last_peered":"2024-09-14T04:35:13.761300+0000","last_clean":"2024-09-14T04:35:13.761300+0000","last_became_active":"2024-09-14T04:35:13.760455+0000","last_became_peered":"2024-09-14T04:35:13.760455+0000","last_unstale":"2024-09-14T04:35:13.761300+0000","last_undegraded":"2024-09-14T04:35:13.761300+0000","last_fullsized":"2024-09-14T04:35:13.761300+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:19:11.363794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":200,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744881+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:35:12.744881+0000","last_peered":"2024-09-14T04:35:12.744881+0000","last_clean":"2024-09-14T04:35:12.744881+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:35:12.744881+0000","last_undegraded":"2024-09-14T04:35:12.744881+0000","last_fullsized":"2024-09-14T04:35:12.744881+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":461,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765788+0000","last_change":"2024-09-14T04:35:13.764757+0000","last_active":"2024-09-14T04:35:13.765788+0000","last_peered":"2024-09-14T04:35:13.765788+0000","last_clean":"2024-09-14T04:35:13.765788+0000","last_became_active":"2024-09-14T04:35:13.763960+0000","last_became_peered":"2024-09-14T04:35:13.763960+0000","last_unstale":"2024-09-14T04:35:13.765788+0000","last_undegraded":"2024-09-14T04:35:13.765788+0000","last_fullsized":"2024-09-14T04:35:13.765788+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:39.810230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"86'149","reported_seq":967,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737252+0000","last_change":"2024-09-14T04:34:02.356446+0000","last_active":"2024-09-14T04:35:12.737252+0000","last_peered":"2024-09-14T04:35:12.737252+0000","last_clean":"2024-09-14T04:35:12.737252+0000","last_became_active":"2024-09-14T04:34:02.356063+0000","last_became_peered":"2024-09-14T04:34:02.356063+0000","last_unstale":"2024-09-14T04:35:12.737252+0000","last_undegraded":"2024-09-14T04:35:12.737252+0000","last_fullsized":"2024-09-14T04:35:12.737252+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:58.219433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:58.219433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:19.836104+0000","scrub_duration":0.0013900010000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":389,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757177+0000","last_change":"2024-09-14T04:34:20.510111+0000","last_active":"2024-09-14T04:35:13.757177+0000","last_peered":"2024-09-14T04:35:13.757177+0000","last_clean":"2024-09-14T04:35:13.757177+0000","last_became_active":"2024-09-14T04:34:20.509761+0000","last_became_peered":"2024-09-14T04:34:20.509761+0000","last_unstale":"2024-09-14T04:35:13.757177+0000","last_undegraded":"2024-09-14T04:35:13.757177+0000","last_fullsized":"2024-09-14T04:35:13.757177+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:36.158698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":60,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757604+0000","last_change":"2024-09-14T04:34:20.509644+0000","last_active":"2024-09-14T04:35:13.757604+0000","last_peered":"2024-09-14T04:35:13.757604+0000","last_clean":"2024-09-14T04:35:13.757604+0000","last_became_active":"2024-09-14T04:34:20.509138+0000","last_became_peered":"2024-09-14T04:34:20.509138+0000","last_unstale":"2024-09-14T04:35:13.757604+0000","last_undegraded":"2024-09-14T04:35:13.757604+0000","last_fullsized":"2024-09-14T04:35:13.757604+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:53:17.819229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"86'149","reported_seq":831,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748152+0000","last_change":"2024-09-14T04:34:05.167904+0000","last_active":"2024-09-14T04:35:13.748152+0000","last_peered":"2024-09-14T04:35:13.748152+0000","last_clean":"2024-09-14T04:35:13.748152+0000","last_became_active":"2024-09-14T04:34:02.356949+0000","last_became_peered":"2024-09-14T04:34:02.356949+0000","last_unstale":"2024-09-14T04:35:13.748152+0000","last_undegraded":"2024-09-14T04:35:13.748152+0000","last_fullsized":"2024-09-14T04:35:13.748152+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:05.167845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:05.167845+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:19.773382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":600,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754660+0000","last_change":"2024-09-14T04:33:59.337174+0000","last_active":"2024-09-14T04:35:13.754660+0000","last_peered":"2024-09-14T04:35:13.754660+0000","last_clean":"2024-09-14T04:35:13.754660+0000","last_became_active":"2024-09-14T04:33:59.336862+0000","last_became_peered":"2024-09-14T04:33:59.336862+0000","last_unstale":"2024-09-14T04:35:13.754660+0000","last_undegraded":"2024-09-14T04:35:13.754660+0000","last_fullsized":"2024-09-14T04:35:13.754660+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:00:23.183522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":861,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753318+0000","last_change":"2024-09-14T04:33:56.348370+0000","last_active":"2024-09-14T04:35:13.753318+0000","last_peered":"2024-09-14T04:35:13.753318+0000","last_clean":"2024-09-14T04:35:13.753318+0000","last_became_active":"2024-09-14T04:33:56.338455+0000","last_became_peered":"2024-09-14T04:33:56.338455+0000","last_unstale":"2024-09-14T04:35:13.753318+0000","last_undegraded":"2024-09-14T04:35:13.753318+0000","last_fullsized":"2024-09-14T04:35:13.753318+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:55:12.047606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":23122,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4620,"ondisk_log_size":4620,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201946,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498560,"kb_used_data":8632,"kb_used_omap":1,"kb_used_meta":1489918,"kb_avail":92246592,"statfs":{"total":95995035648,"available":94460510208,"internally_reserved":0,"allocated":8839168,"data_stored":5081895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1525676491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463584,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1217332,"kb_used_data":8676,"kb_used_omap":1,"kb_used_meta":1208638,"kb_avail":92527820,"statfs":{"total":95995035648,"available":94748487680,"internally_reserved":0,"allocated":8884224,"data_stored":5146206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1237645767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834257,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":635692,"kb_used_data":8612,"kb_used_omap":5,"kb_used_meta":627066,"kb_avail":93109460,"statfs":{"total":95995035648,"available":95344087040,"internally_reserved":0,"allocated":8818688,"data_stored":5077313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":642115645},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986856,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722876,"kb_used_data":8364,"kb_used_omap":1,"kb_used_meta":1714494,"kb_avail":92022276,"statfs":{"total":95995035648,"available":94230810624,"internally_reserved":0,"allocated":8564736,"data_stored":4818864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1755642310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748884,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266284,"kb_used_data":8668,"kb_used_omap":4,"kb_used_meta":1257595,"kb_avail":92478868,"statfs":{"total":95995035648,"available":94698360832,"internally_reserved":0,"allocated":8876032,"data_stored":5142017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1287777836},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605287105,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650028,"kb_used_data":8612,"kb_used_omap":10,"kb_used_meta":641397,"kb_avail":93095124,"statfs":{"total":95995035648,"available":95329406976,"internally_reserved":0,"allocated":8818688,"data_stored":5077551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":656791127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468428,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191012,"kb_used_data":8344,"kb_used_omap":5,"kb_used_meta":1182650,"kb_avail":92554140,"statfs":{"total":95995035648,"available":94775439360,"internally_reserved":0,"allocated":8544256,"data_stored":4812846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1211033951},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342543,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":996524,"kb_used_data":8672,"kb_used_omap":10,"kb_used_meta":987829,"kb_avail":92748628,"statfs":{"total":95995035648,"available":94974595072,"internally_reserved":0,"allocated":8880128,"data_stored":5142531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":1011537106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:36:04.048 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 5626d550-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:36:04.265 INFO:teuthology.orchestra.run.smithi001.stderr:Inferring config /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:36:04.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:04 smithi001 ceph-mon[112165]: from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:04.970 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:04 smithi001 ceph-mon[112165]: pgmap v1862: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:04 smithi001 ceph-mon[107399]: from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:04.971 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:04 smithi001 ceph-mon[107399]: pgmap v1862: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:04 smithi012 ceph-mon[101180]: from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:05.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:04 smithi012 ceph-mon[101180]: pgmap v1862: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:06.017 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:05 smithi001 ceph-mon[107399]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:06.017 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:05 smithi001 ceph-mon[112165]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:06.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:05 smithi012 ceph-mon[101180]: from='mgr.34104 172.21.15.1:0/667857939' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:06.216 INFO:teuthology.orchestra.run.smithi001.stdout: 2024-09-14T04:36:06.217 INFO:teuthology.orchestra.run.smithi001.stderr:dumped all 2024-09-14T04:36:06.754 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:06 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: ::ffff:172.21.15.12 - - [14/Sep/2024:04:36:06] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.51.0" 2024-09-14T04:36:06.889 INFO:teuthology.orchestra.run.smithi001.stdout:{"pg_ready":true,"pg_map":{"version":1863,"stamp":"2024-09-14T04:36:05.695161+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":23122,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4679,"ondisk_log_size":4679,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9178308,"kb_used_data":68580,"kb_used_omap":41,"kb_used_meta":9109590,"kb_avail":740782908,"statfs":{"total":767960285184,"available":758561697792,"internally_reserved":0,"allocated":70225920,"data_stored":40299223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42935,"internal_metadata":9328220233},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003109"},"pg_stats":[{"pgid":"1.11","version":"86'149","reported_seq":983,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752277+0000","last_change":"2024-09-14T04:25:28.613509+0000","last_active":"2024-09-14T04:35:13.752277+0000","last_peered":"2024-09-14T04:35:13.752277+0000","last_clean":"2024-09-14T04:35:13.752277+0000","last_became_active":"2024-09-14T04:25:28.612918+0000","last_became_peered":"2024-09-14T04:25:28.612918+0000","last_unstale":"2024-09-14T04:35:13.752277+0000","last_undegraded":"2024-09-14T04:35:13.752277+0000","last_fullsized":"2024-09-14T04:35:13.752277+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:24:49.775618+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:24:49.775618+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:33:13.896667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757333+0000","last_change":"2024-09-14T04:34:20.656491+0000","last_active":"2024-09-14T04:35:13.757333+0000","last_peered":"2024-09-14T04:35:13.757333+0000","last_clean":"2024-09-14T04:35:13.757333+0000","last_became_active":"2024-09-14T04:34:20.656169+0000","last_became_peered":"2024-09-14T04:34:20.656169+0000","last_unstale":"2024-09-14T04:35:13.757333+0000","last_undegraded":"2024-09-14T04:35:13.757333+0000","last_fullsized":"2024-09-14T04:35:13.757333+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:07:08.833533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":195,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737355+0000","last_change":"2024-09-14T04:33:56.345878+0000","last_active":"2024-09-14T04:35:12.737355+0000","last_peered":"2024-09-14T04:35:12.737355+0000","last_clean":"2024-09-14T04:35:12.737355+0000","last_became_active":"2024-09-14T04:33:56.334984+0000","last_became_peered":"2024-09-14T04:33:56.334984+0000","last_unstale":"2024-09-14T04:35:12.737355+0000","last_undegraded":"2024-09-14T04:35:12.737355+0000","last_fullsized":"2024-09-14T04:35:12.737355+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:26.707796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"86'149","reported_seq":859,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.764814+0000","last_change":"2024-09-14T04:35:13.763098+0000","last_active":"2024-09-14T04:35:13.764814+0000","last_peered":"2024-09-14T04:35:13.764814+0000","last_clean":"2024-09-14T04:35:13.764814+0000","last_became_active":"2024-09-14T04:35:13.762699+0000","last_became_peered":"2024-09-14T04:35:13.762699+0000","last_unstale":"2024-09-14T04:35:13.764814+0000","last_undegraded":"2024-09-14T04:35:13.764814+0000","last_fullsized":"2024-09-14T04:35:13.764814+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:15.602726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:15.602726+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:16.784038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.762911+0000","last_change":"2024-09-14T04:35:13.762453+0000","last_active":"2024-09-14T04:35:13.762911+0000","last_peered":"2024-09-14T04:35:13.762911+0000","last_clean":"2024-09-14T04:35:13.762911+0000","last_became_active":"2024-09-14T04:35:13.762116+0000","last_became_peered":"2024-09-14T04:35:13.762116+0000","last_unstale":"2024-09-14T04:35:13.762911+0000","last_undegraded":"2024-09-14T04:35:13.762911+0000","last_fullsized":"2024-09-14T04:35:13.762911+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:59.591678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":803,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752221+0000","last_change":"2024-09-14T04:27:20.244818+0000","last_active":"2024-09-14T04:35:13.752221+0000","last_peered":"2024-09-14T04:35:13.752221+0000","last_clean":"2024-09-14T04:35:13.752221+0000","last_became_active":"2024-09-14T04:27:20.244519+0000","last_became_peered":"2024-09-14T04:27:20.244519+0000","last_unstale":"2024-09-14T04:35:13.752221+0000","last_undegraded":"2024-09-14T04:35:13.752221+0000","last_fullsized":"2024-09-14T04:35:13.752221+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:28.651548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"86'149","reported_seq":965,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748125+0000","last_change":"2024-09-14T04:34:02.358526+0000","last_active":"2024-09-14T04:35:13.748125+0000","last_peered":"2024-09-14T04:35:13.748125+0000","last_clean":"2024-09-14T04:35:13.748125+0000","last_became_active":"2024-09-14T04:34:02.357982+0000","last_became_peered":"2024-09-14T04:34:02.357982+0000","last_unstale":"2024-09-14T04:35:13.748125+0000","last_undegraded":"2024-09-14T04:35:13.748125+0000","last_fullsized":"2024-09-14T04:35:13.748125+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.970667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.970667+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:02.769602+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":789,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752971+0000","last_change":"2024-09-14T04:27:20.248783+0000","last_active":"2024-09-14T04:35:13.752971+0000","last_peered":"2024-09-14T04:35:13.752971+0000","last_clean":"2024-09-14T04:35:13.752971+0000","last_became_active":"2024-09-14T04:27:20.247981+0000","last_became_peered":"2024-09-14T04:27:20.247981+0000","last_unstale":"2024-09-14T04:35:13.752971+0000","last_undegraded":"2024-09-14T04:35:13.752971+0000","last_fullsized":"2024-09-14T04:35:13.752971+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:24.895186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.736773+0000","last_change":"2024-09-14T04:28:19.258127+0000","last_active":"2024-09-14T04:35:12.736773+0000","last_peered":"2024-09-14T04:35:12.736773+0000","last_clean":"2024-09-14T04:35:12.736773+0000","last_became_active":"2024-09-14T04:28:19.257230+0000","last_became_peered":"2024-09-14T04:28:19.257230+0000","last_unstale":"2024-09-14T04:35:12.736773+0000","last_undegraded":"2024-09-14T04:35:12.736773+0000","last_fullsized":"2024-09-14T04:35:12.736773+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:46.864460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"86'149","reported_seq":712,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.764734+0000","last_change":"2024-09-14T04:35:13.763616+0000","last_active":"2024-09-14T04:35:13.764734+0000","last_peered":"2024-09-14T04:35:13.764734+0000","last_clean":"2024-09-14T04:35:13.764734+0000","last_became_active":"2024-09-14T04:35:13.763083+0000","last_became_peered":"2024-09-14T04:35:13.763083+0000","last_unstale":"2024-09-14T04:35:13.764734+0000","last_undegraded":"2024-09-14T04:35:13.764734+0000","last_fullsized":"2024-09-14T04:35:13.764734+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:14.646450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:14.646450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:47:33.126366+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":397,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757681+0000","last_change":"2024-09-14T04:34:53.787766+0000","last_active":"2024-09-14T04:35:13.757681+0000","last_peered":"2024-09-14T04:35:13.757681+0000","last_clean":"2024-09-14T04:35:13.757681+0000","last_became_active":"2024-09-14T04:34:53.787249+0000","last_became_peered":"2024-09-14T04:34:53.787249+0000","last_unstale":"2024-09-14T04:35:13.757681+0000","last_undegraded":"2024-09-14T04:35:13.757681+0000","last_fullsized":"2024-09-14T04:35:13.757681+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:16.891079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":150,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.758978+0000","last_change":"2024-09-14T04:28:19.288301+0000","last_active":"2024-09-14T04:35:13.758978+0000","last_peered":"2024-09-14T04:35:13.758978+0000","last_clean":"2024-09-14T04:35:13.758978+0000","last_became_active":"2024-09-14T04:28:19.260682+0000","last_became_peered":"2024-09-14T04:28:19.260682+0000","last_unstale":"2024-09-14T04:35:13.758978+0000","last_undegraded":"2024-09-14T04:35:13.758978+0000","last_fullsized":"2024-09-14T04:35:13.758978+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:05.785810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"86'149","reported_seq":921,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737783+0000","last_change":"2024-09-14T04:34:02.356617+0000","last_active":"2024-09-14T04:35:12.737783+0000","last_peered":"2024-09-14T04:35:12.737783+0000","last_clean":"2024-09-14T04:35:12.737783+0000","last_became_active":"2024-09-14T04:34:02.356019+0000","last_became_peered":"2024-09-14T04:34:02.356019+0000","last_unstale":"2024-09-14T04:35:12.737783+0000","last_undegraded":"2024-09-14T04:35:12.737783+0000","last_fullsized":"2024-09-14T04:35:12.737783+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:51.131145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:51.131145+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:16:18.362346+0000","scrub_duration":0.001864201,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":163,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.738277+0000","last_change":"2024-09-14T04:27:20.359458+0000","last_active":"2024-09-14T04:35:12.738277+0000","last_peered":"2024-09-14T04:35:12.738277+0000","last_clean":"2024-09-14T04:35:12.738277+0000","last_became_active":"2024-09-14T04:27:20.247560+0000","last_became_peered":"2024-09-14T04:27:20.247560+0000","last_unstale":"2024-09-14T04:35:12.738277+0000","last_undegraded":"2024-09-14T04:35:12.738277+0000","last_fullsized":"2024-09-14T04:35:12.738277+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:31.173432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765402+0000","last_change":"2024-09-14T04:35:13.764216+0000","last_active":"2024-09-14T04:35:13.765402+0000","last_peered":"2024-09-14T04:35:13.765402+0000","last_clean":"2024-09-14T04:35:13.765402+0000","last_became_active":"2024-09-14T04:35:13.763708+0000","last_became_peered":"2024-09-14T04:35:13.763708+0000","last_unstale":"2024-09-14T04:35:13.765402+0000","last_undegraded":"2024-09-14T04:35:13.765402+0000","last_fullsized":"2024-09-14T04:35:13.765402+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:43.963537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"86'149","reported_seq":1357,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757453+0000","last_change":"2024-09-14T04:33:49.259743+0000","last_active":"2024-09-14T04:35:13.757453+0000","last_peered":"2024-09-14T04:35:13.757453+0000","last_clean":"2024-09-14T04:35:13.757453+0000","last_became_active":"2024-09-14T04:33:48.758490+0000","last_became_peered":"2024-09-14T04:33:48.758490+0000","last_unstale":"2024-09-14T04:35:13.757453+0000","last_undegraded":"2024-09-14T04:35:13.757453+0000","last_fullsized":"2024-09-14T04:35:13.757453+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.259685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.259685+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:56.085704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":189,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.738178+0000","last_change":"2024-09-14T04:33:59.349119+0000","last_active":"2024-09-14T04:35:12.738178+0000","last_peered":"2024-09-14T04:35:12.738178+0000","last_clean":"2024-09-14T04:35:12.738178+0000","last_became_active":"2024-09-14T04:33:59.336323+0000","last_became_peered":"2024-09-14T04:33:59.336323+0000","last_unstale":"2024-09-14T04:35:12.738178+0000","last_undegraded":"2024-09-14T04:35:12.738178+0000","last_fullsized":"2024-09-14T04:35:12.738178+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:51.489919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":644,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744304+0000","last_change":"2024-09-14T04:33:59.335137+0000","last_active":"2024-09-14T04:35:12.744304+0000","last_peered":"2024-09-14T04:35:12.744304+0000","last_clean":"2024-09-14T04:35:12.744304+0000","last_became_active":"2024-09-14T04:33:59.334808+0000","last_became_peered":"2024-09-14T04:33:59.334808+0000","last_unstale":"2024-09-14T04:35:12.744304+0000","last_undegraded":"2024-09-14T04:35:12.744304+0000","last_fullsized":"2024-09-14T04:35:12.744304+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:15:33.958948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"86'149","reported_seq":539,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748009+0000","last_change":"2024-09-14T04:28:22.703089+0000","last_active":"2024-09-14T04:35:12.748009+0000","last_peered":"2024-09-14T04:35:12.748009+0000","last_clean":"2024-09-14T04:35:12.748009+0000","last_became_active":"2024-09-14T04:28:22.702760+0000","last_became_peered":"2024-09-14T04:28:22.702760+0000","last_unstale":"2024-09-14T04:35:12.748009+0000","last_undegraded":"2024-09-14T04:35:12.748009+0000","last_fullsized":"2024-09-14T04:35:12.748009+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:21.769016+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:19:21.769016+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:33.566946+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":176,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737970+0000","last_change":"2024-09-14T04:28:19.265783+0000","last_active":"2024-09-14T04:35:12.737970+0000","last_peered":"2024-09-14T04:35:12.737970+0000","last_clean":"2024-09-14T04:35:12.737970+0000","last_became_active":"2024-09-14T04:28:19.257596+0000","last_became_peered":"2024-09-14T04:28:19.257596+0000","last_unstale":"2024-09-14T04:35:12.737970+0000","last_undegraded":"2024-09-14T04:35:12.737970+0000","last_fullsized":"2024-09-14T04:35:12.737970+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:22.427004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748054+0000","last_change":"2024-09-14T04:33:59.345940+0000","last_active":"2024-09-14T04:35:12.748054+0000","last_peered":"2024-09-14T04:35:12.748054+0000","last_clean":"2024-09-14T04:35:12.748054+0000","last_became_active":"2024-09-14T04:33:59.345690+0000","last_became_peered":"2024-09-14T04:33:59.345690+0000","last_unstale":"2024-09-14T04:35:12.748054+0000","last_undegraded":"2024-09-14T04:35:12.748054+0000","last_fullsized":"2024-09-14T04:35:12.748054+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:19.006679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"86'149","reported_seq":617,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747064+0000","last_change":"2024-09-14T04:33:49.523592+0000","last_active":"2024-09-14T04:35:12.747064+0000","last_peered":"2024-09-14T04:35:12.747064+0000","last_clean":"2024-09-14T04:35:12.747064+0000","last_became_active":"2024-09-14T04:33:48.759868+0000","last_became_peered":"2024-09-14T04:33:48.759868+0000","last_unstale":"2024-09-14T04:35:12.747064+0000","last_undegraded":"2024-09-14T04:35:12.747064+0000","last_fullsized":"2024-09-14T04:35:12.747064+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.523485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.523485+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:38.218380+0000","scrub_duration":0.00154894,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":462,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765600+0000","last_change":"2024-09-14T04:35:13.765040+0000","last_active":"2024-09-14T04:35:13.765600+0000","last_peered":"2024-09-14T04:35:13.765600+0000","last_clean":"2024-09-14T04:35:13.765600+0000","last_became_active":"2024-09-14T04:35:13.764527+0000","last_became_peered":"2024-09-14T04:35:13.764527+0000","last_unstale":"2024-09-14T04:35:13.765600+0000","last_undegraded":"2024-09-14T04:35:13.765600+0000","last_fullsized":"2024-09-14T04:35:13.765600+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:39.654987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":790,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753033+0000","last_change":"2024-09-14T04:33:59.341074+0000","last_active":"2024-09-14T04:35:13.753033+0000","last_peered":"2024-09-14T04:35:13.753033+0000","last_clean":"2024-09-14T04:35:13.753033+0000","last_became_active":"2024-09-14T04:33:59.340717+0000","last_became_peered":"2024-09-14T04:33:59.340717+0000","last_unstale":"2024-09-14T04:35:13.753033+0000","last_undegraded":"2024-09-14T04:35:13.753033+0000","last_fullsized":"2024-09-14T04:35:13.753033+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:22.803669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"86'149","reported_seq":981,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749278+0000","last_change":"2024-09-14T04:33:59.328955+0000","last_active":"2024-09-14T04:35:13.749278+0000","last_peered":"2024-09-14T04:35:13.749278+0000","last_clean":"2024-09-14T04:35:13.749278+0000","last_became_active":"2024-09-14T04:33:59.328562+0000","last_became_peered":"2024-09-14T04:33:59.328562+0000","last_unstale":"2024-09-14T04:35:13.749278+0000","last_undegraded":"2024-09-14T04:35:13.749278+0000","last_fullsized":"2024-09-14T04:35:13.749278+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.165396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.165396+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:06.475748+0000","scrub_duration":0.0017422690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":64,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757301+0000","last_change":"2024-09-14T04:34:20.509266+0000","last_active":"2024-09-14T04:35:13.757301+0000","last_peered":"2024-09-14T04:35:13.757301+0000","last_clean":"2024-09-14T04:35:13.757301+0000","last_became_active":"2024-09-14T04:34:20.508582+0000","last_became_peered":"2024-09-14T04:34:20.508582+0000","last_unstale":"2024-09-14T04:35:13.757301+0000","last_undegraded":"2024-09-14T04:35:13.757301+0000","last_fullsized":"2024-09-14T04:35:13.757301+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:36:32.195051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":628,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746848+0000","last_change":"2024-09-14T04:32:52.819885+0000","last_active":"2024-09-14T04:35:12.746848+0000","last_peered":"2024-09-14T04:35:12.746848+0000","last_clean":"2024-09-14T04:35:12.746848+0000","last_became_active":"2024-09-14T04:32:52.819034+0000","last_became_peered":"2024-09-14T04:32:52.819034+0000","last_unstale":"2024-09-14T04:35:12.746848+0000","last_undegraded":"2024-09-14T04:35:12.746848+0000","last_fullsized":"2024-09-14T04:35:12.746848+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:52.327969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"86'149","reported_seq":750,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:22.093913+0000","last_change":"2024-09-14T04:35:22.094000+0000","last_active":"2024-09-14T04:35:22.093913+0000","last_peered":"2024-09-14T04:35:22.093913+0000","last_clean":"2024-09-14T04:35:22.093913+0000","last_became_active":"2024-09-14T04:35:13.759137+0000","last_became_peered":"2024-09-14T04:35:13.759137+0000","last_unstale":"2024-09-14T04:35:22.093913+0000","last_undegraded":"2024-09-14T04:35:22.093913+0000","last_fullsized":"2024-09-14T04:35:22.093913+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:22.093897+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:22.093897+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:34.169748+0000","scrub_duration":0.0019979960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748164+0000","last_change":"2024-09-14T04:33:59.345684+0000","last_active":"2024-09-14T04:35:12.748164+0000","last_peered":"2024-09-14T04:35:12.748164+0000","last_clean":"2024-09-14T04:35:12.748164+0000","last_became_active":"2024-09-14T04:33:59.345345+0000","last_became_peered":"2024-09-14T04:33:59.345345+0000","last_unstale":"2024-09-14T04:35:12.748164+0000","last_undegraded":"2024-09-14T04:35:12.748164+0000","last_fullsized":"2024-09-14T04:35:12.748164+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:43:41.565785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":236,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747096+0000","last_change":"2024-09-14T04:33:59.341288+0000","last_active":"2024-09-14T04:35:12.747096+0000","last_peered":"2024-09-14T04:35:12.747096+0000","last_clean":"2024-09-14T04:35:12.747096+0000","last_became_active":"2024-09-14T04:33:59.340994+0000","last_became_peered":"2024-09-14T04:33:59.340994+0000","last_unstale":"2024-09-14T04:35:12.747096+0000","last_undegraded":"2024-09-14T04:35:12.747096+0000","last_fullsized":"2024-09-14T04:35:12.747096+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:05.725236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"86'149","reported_seq":1047,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747866+0000","last_change":"2024-09-14T04:34:02.358731+0000","last_active":"2024-09-14T04:35:13.747866+0000","last_peered":"2024-09-14T04:35:13.747866+0000","last_clean":"2024-09-14T04:35:13.747866+0000","last_became_active":"2024-09-14T04:34:02.357980+0000","last_became_peered":"2024-09-14T04:34:02.357980+0000","last_unstale":"2024-09-14T04:35:13.747866+0000","last_undegraded":"2024-09-14T04:35:13.747866+0000","last_fullsized":"2024-09-14T04:35:13.747866+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.113032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.113032+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:00:06.066978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":237,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754783+0000","last_change":"2024-09-14T04:33:56.336399+0000","last_active":"2024-09-14T04:35:13.754783+0000","last_peered":"2024-09-14T04:35:13.754783+0000","last_clean":"2024-09-14T04:35:13.754783+0000","last_became_active":"2024-09-14T04:33:56.336070+0000","last_became_peered":"2024-09-14T04:33:56.336070+0000","last_unstale":"2024-09-14T04:35:13.754783+0000","last_undegraded":"2024-09-14T04:35:13.754783+0000","last_fullsized":"2024-09-14T04:35:13.754783+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:08.324552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":634,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747258+0000","last_change":"2024-09-14T04:33:59.345358+0000","last_active":"2024-09-14T04:35:12.747258+0000","last_peered":"2024-09-14T04:35:12.747258+0000","last_clean":"2024-09-14T04:35:12.747258+0000","last_became_active":"2024-09-14T04:33:59.345038+0000","last_became_peered":"2024-09-14T04:33:59.345038+0000","last_unstale":"2024-09-14T04:35:12.747258+0000","last_undegraded":"2024-09-14T04:35:12.747258+0000","last_fullsized":"2024-09-14T04:35:12.747258+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:25.738959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"86'149","reported_seq":738,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:32.302067+0000","last_change":"2024-09-14T04:35:32.302153+0000","last_active":"2024-09-14T04:35:32.302067+0000","last_peered":"2024-09-14T04:35:32.302067+0000","last_clean":"2024-09-14T04:35:32.302067+0000","last_became_active":"2024-09-14T04:35:13.763207+0000","last_became_peered":"2024-09-14T04:35:13.763207+0000","last_unstale":"2024-09-14T04:35:32.302067+0000","last_undegraded":"2024-09-14T04:35:32.302067+0000","last_fullsized":"2024-09-14T04:35:32.302067+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:32.302048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:32.302048+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:10.908568+0000","scrub_duration":0.001810346,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":383,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756961+0000","last_change":"2024-09-14T04:33:59.340304+0000","last_active":"2024-09-14T04:35:13.756961+0000","last_peered":"2024-09-14T04:35:13.756961+0000","last_clean":"2024-09-14T04:35:13.756961+0000","last_became_active":"2024-09-14T04:33:59.339996+0000","last_became_peered":"2024-09-14T04:33:59.339996+0000","last_unstale":"2024-09-14T04:35:13.756961+0000","last_undegraded":"2024-09-14T04:35:13.756961+0000","last_fullsized":"2024-09-14T04:35:13.756961+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:18:11.254308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748038+0000","last_change":"2024-09-14T04:33:59.334638+0000","last_active":"2024-09-14T04:35:13.748038+0000","last_peered":"2024-09-14T04:35:13.748038+0000","last_clean":"2024-09-14T04:35:13.748038+0000","last_became_active":"2024-09-14T04:33:59.334323+0000","last_became_peered":"2024-09-14T04:33:59.334323+0000","last_unstale":"2024-09-14T04:35:13.748038+0000","last_undegraded":"2024-09-14T04:35:13.748038+0000","last_fullsized":"2024-09-14T04:35:13.748038+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:20.689612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"86'149","reported_seq":998,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749486+0000","last_change":"2024-09-14T04:28:22.697308+0000","last_active":"2024-09-14T04:35:13.749486+0000","last_peered":"2024-09-14T04:35:13.749486+0000","last_clean":"2024-09-14T04:35:13.749486+0000","last_became_active":"2024-09-14T04:28:22.696943+0000","last_became_peered":"2024-09-14T04:28:22.696943+0000","last_unstale":"2024-09-14T04:35:13.749486+0000","last_undegraded":"2024-09-14T04:35:13.749486+0000","last_fullsized":"2024-09-14T04:35:13.749486+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:16.586635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:16.586635+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:33:01.169534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":456,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.762507+0000","last_change":"2024-09-14T04:35:13.761860+0000","last_active":"2024-09-14T04:35:13.762507+0000","last_peered":"2024-09-14T04:35:13.762507+0000","last_clean":"2024-09-14T04:35:13.762507+0000","last_became_active":"2024-09-14T04:35:13.761215+0000","last_became_peered":"2024-09-14T04:35:13.761215+0000","last_unstale":"2024-09-14T04:35:13.762507+0000","last_undegraded":"2024-09-14T04:35:13.762507+0000","last_fullsized":"2024-09-14T04:35:13.762507+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:07.554220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757082+0000","last_change":"2024-09-14T04:34:20.507274+0000","last_active":"2024-09-14T04:35:13.757082+0000","last_peered":"2024-09-14T04:35:13.757082+0000","last_clean":"2024-09-14T04:35:13.757082+0000","last_became_active":"2024-09-14T04:34:20.506964+0000","last_became_peered":"2024-09-14T04:34:20.506964+0000","last_unstale":"2024-09-14T04:35:13.757082+0000","last_undegraded":"2024-09-14T04:35:13.757082+0000","last_fullsized":"2024-09-14T04:35:13.757082+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:26:57.758549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746320+0000","last_change":"2024-09-14T04:32:52.817519+0000","last_active":"2024-09-14T04:35:12.746320+0000","last_peered":"2024-09-14T04:35:12.746320+0000","last_clean":"2024-09-14T04:35:12.746320+0000","last_became_active":"2024-09-14T04:32:52.817207+0000","last_became_peered":"2024-09-14T04:32:52.817207+0000","last_unstale":"2024-09-14T04:35:12.746320+0000","last_undegraded":"2024-09-14T04:35:12.746320+0000","last_fullsized":"2024-09-14T04:35:12.746320+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:35:43.967076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"86'149","reported_seq":469,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748000+0000","last_change":"2024-09-14T04:34:03.164615+0000","last_active":"2024-09-14T04:35:13.748000+0000","last_peered":"2024-09-14T04:35:13.748000+0000","last_clean":"2024-09-14T04:35:13.748000+0000","last_became_active":"2024-09-14T04:34:02.355231+0000","last_became_peered":"2024-09-14T04:34:02.355231+0000","last_unstale":"2024-09-14T04:35:13.748000+0000","last_undegraded":"2024-09-14T04:35:13.748000+0000","last_fullsized":"2024-09-14T04:35:13.748000+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:03.164560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:03.164560+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:38:27.410822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":178,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749382+0000","last_change":"2024-09-14T04:33:59.334578+0000","last_active":"2024-09-14T04:35:13.749382+0000","last_peered":"2024-09-14T04:35:13.749382+0000","last_clean":"2024-09-14T04:35:13.749382+0000","last_became_active":"2024-09-14T04:33:59.334246+0000","last_became_peered":"2024-09-14T04:33:59.334246+0000","last_unstale":"2024-09-14T04:35:13.749382+0000","last_undegraded":"2024-09-14T04:35:13.749382+0000","last_fullsized":"2024-09-14T04:35:13.749382+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:01.401424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060361599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1026,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752279+0000","last_change":"2024-09-14T04:25:35.563357+0000","last_active":"2024-09-14T04:35:13.752279+0000","last_peered":"2024-09-14T04:35:13.752279+0000","last_clean":"2024-09-14T04:35:13.752279+0000","last_became_active":"2024-09-14T04:25:35.562197+0000","last_became_peered":"2024-09-14T04:25:35.562197+0000","last_unstale":"2024-09-14T04:35:13.752279+0000","last_undegraded":"2024-09-14T04:35:13.752279+0000","last_fullsized":"2024-09-14T04:35:13.752279+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:10.426666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":815,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753056+0000","last_change":"2024-09-14T04:33:59.335406+0000","last_active":"2024-09-14T04:35:13.753056+0000","last_peered":"2024-09-14T04:35:13.753056+0000","last_clean":"2024-09-14T04:35:13.753056+0000","last_became_active":"2024-09-14T04:33:59.334973+0000","last_became_peered":"2024-09-14T04:33:59.334973+0000","last_unstale":"2024-09-14T04:35:13.753056+0000","last_undegraded":"2024-09-14T04:35:13.753056+0000","last_fullsized":"2024-09-14T04:35:13.753056+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:44:28.459636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"86'149","reported_seq":856,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749342+0000","last_change":"2024-09-14T04:27:23.245485+0000","last_active":"2024-09-14T04:35:13.749342+0000","last_peered":"2024-09-14T04:35:13.749342+0000","last_clean":"2024-09-14T04:35:13.749342+0000","last_became_active":"2024-09-14T04:27:23.245124+0000","last_became_peered":"2024-09-14T04:27:23.245124+0000","last_unstale":"2024-09-14T04:35:13.749342+0000","last_undegraded":"2024-09-14T04:35:13.749342+0000","last_fullsized":"2024-09-14T04:35:13.749342+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:22:19.657759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:22:19.657759+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:03.702563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":63,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753928+0000","last_change":"2024-09-14T04:33:56.335336+0000","last_active":"2024-09-14T04:35:13.753928+0000","last_peered":"2024-09-14T04:35:13.753928+0000","last_clean":"2024-09-14T04:35:13.753928+0000","last_became_active":"2024-09-14T04:33:56.335029+0000","last_became_peered":"2024-09-14T04:33:56.335029+0000","last_unstale":"2024-09-14T04:35:13.753928+0000","last_undegraded":"2024-09-14T04:35:13.753928+0000","last_fullsized":"2024-09-14T04:35:13.753928+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:33:31.715946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081915500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747704+0000","last_change":"2024-09-14T04:33:59.339238+0000","last_active":"2024-09-14T04:35:13.747704+0000","last_peered":"2024-09-14T04:35:13.747704+0000","last_clean":"2024-09-14T04:35:13.747704+0000","last_became_active":"2024-09-14T04:33:59.338916+0000","last_became_peered":"2024-09-14T04:33:59.338916+0000","last_unstale":"2024-09-14T04:35:13.747704+0000","last_undegraded":"2024-09-14T04:35:13.747704+0000","last_fullsized":"2024-09-14T04:35:13.747704+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:00.746903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747923+0000","last_change":"2024-09-14T04:33:59.333665+0000","last_active":"2024-09-14T04:35:13.747923+0000","last_peered":"2024-09-14T04:35:13.747923+0000","last_clean":"2024-09-14T04:35:13.747923+0000","last_became_active":"2024-09-14T04:33:59.331169+0000","last_became_peered":"2024-09-14T04:33:59.331169+0000","last_unstale":"2024-09-14T04:35:13.747923+0000","last_undegraded":"2024-09-14T04:35:13.747923+0000","last_fullsized":"2024-09-14T04:35:13.747923+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:53:01.798897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"86'149","reported_seq":732,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.759398+0000","last_change":"2024-09-14T04:35:13.759000+0000","last_active":"2024-09-14T04:35:13.759398+0000","last_peered":"2024-09-14T04:35:13.759398+0000","last_clean":"2024-09-14T04:35:13.759398+0000","last_became_active":"2024-09-14T04:35:13.758415+0000","last_became_peered":"2024-09-14T04:35:13.758415+0000","last_unstale":"2024-09-14T04:35:13.759398+0000","last_undegraded":"2024-09-14T04:35:13.759398+0000","last_fullsized":"2024-09-14T04:35:13.759398+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":4,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:29:18.697092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:29:18.697092+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:54:04.352518+0000","scrub_duration":0.002345757,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":440,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765465+0000","last_change":"2024-09-14T04:35:13.764189+0000","last_active":"2024-09-14T04:35:13.765465+0000","last_peered":"2024-09-14T04:35:13.765465+0000","last_clean":"2024-09-14T04:35:13.765465+0000","last_became_active":"2024-09-14T04:35:13.763794+0000","last_became_peered":"2024-09-14T04:35:13.763794+0000","last_unstale":"2024-09-14T04:35:13.765465+0000","last_undegraded":"2024-09-14T04:35:13.765465+0000","last_fullsized":"2024-09-14T04:35:13.765465+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:29:34.136506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087772100000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":619,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746545+0000","last_change":"2024-09-14T04:28:19.267097+0000","last_active":"2024-09-14T04:35:12.746545+0000","last_peered":"2024-09-14T04:35:12.746545+0000","last_clean":"2024-09-14T04:35:12.746545+0000","last_became_active":"2024-09-14T04:28:19.266725+0000","last_became_peered":"2024-09-14T04:28:19.266725+0000","last_unstale":"2024-09-14T04:35:12.746545+0000","last_undegraded":"2024-09-14T04:35:12.746545+0000","last_fullsized":"2024-09-14T04:35:12.746545+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:20:21.868313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748114+0000","last_change":"2024-09-14T04:33:59.333265+0000","last_active":"2024-09-14T04:35:13.748114+0000","last_peered":"2024-09-14T04:35:13.748114+0000","last_clean":"2024-09-14T04:35:13.748114+0000","last_became_active":"2024-09-14T04:33:59.332915+0000","last_became_peered":"2024-09-14T04:33:59.332915+0000","last_unstale":"2024-09-14T04:35:13.748114+0000","last_undegraded":"2024-09-14T04:35:13.748114+0000","last_fullsized":"2024-09-14T04:35:13.748114+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:58:07.942027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"86'149","reported_seq":1004,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747816+0000","last_change":"2024-09-14T04:33:49.324876+0000","last_active":"2024-09-14T04:35:12.747816+0000","last_peered":"2024-09-14T04:35:12.747816+0000","last_clean":"2024-09-14T04:35:12.747816+0000","last_became_active":"2024-09-14T04:33:48.761757+0000","last_became_peered":"2024-09-14T04:33:48.761757+0000","last_unstale":"2024-09-14T04:35:12.747816+0000","last_undegraded":"2024-09-14T04:35:12.747816+0000","last_fullsized":"2024-09-14T04:35:12.747816+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:49.324749+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:33:49.324749+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:30.576200+0000","scrub_duration":0.001637119,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":602,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.759450+0000","last_change":"2024-09-14T04:35:13.758908+0000","last_active":"2024-09-14T04:35:13.759450+0000","last_peered":"2024-09-14T04:35:13.759450+0000","last_clean":"2024-09-14T04:35:13.759450+0000","last_became_active":"2024-09-14T04:35:13.758337+0000","last_became_peered":"2024-09-14T04:35:13.758337+0000","last_unstale":"2024-09-14T04:35:13.759450+0000","last_undegraded":"2024-09-14T04:35:13.759450+0000","last_fullsized":"2024-09-14T04:35:13.759450+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:29:38.259259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748263+0000","last_change":"2024-09-14T04:33:59.334159+0000","last_active":"2024-09-14T04:35:13.748263+0000","last_peered":"2024-09-14T04:35:13.748263+0000","last_clean":"2024-09-14T04:35:13.748263+0000","last_became_active":"2024-09-14T04:33:59.333204+0000","last_became_peered":"2024-09-14T04:33:59.333204+0000","last_unstale":"2024-09-14T04:35:13.748263+0000","last_undegraded":"2024-09-14T04:35:13.748263+0000","last_fullsized":"2024-09-14T04:35:13.748263+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:39:16.678016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030878600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748108+0000","last_change":"2024-09-14T04:33:59.335699+0000","last_active":"2024-09-14T04:35:13.748108+0000","last_peered":"2024-09-14T04:35:13.748108+0000","last_clean":"2024-09-14T04:35:13.748108+0000","last_became_active":"2024-09-14T04:33:59.335204+0000","last_became_peered":"2024-09-14T04:33:59.335204+0000","last_unstale":"2024-09-14T04:35:13.748108+0000","last_undegraded":"2024-09-14T04:35:13.748108+0000","last_fullsized":"2024-09-14T04:35:13.748108+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:14.368649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"86'149","reported_seq":934,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748323+0000","last_change":"2024-09-14T04:33:50.279734+0000","last_active":"2024-09-14T04:35:12.748323+0000","last_peered":"2024-09-14T04:35:12.748323+0000","last_clean":"2024-09-14T04:35:12.748323+0000","last_became_active":"2024-09-14T04:33:48.761487+0000","last_became_peered":"2024-09-14T04:33:48.761487+0000","last_unstale":"2024-09-14T04:35:12.748323+0000","last_undegraded":"2024-09-14T04:35:12.748323+0000","last_fullsized":"2024-09-14T04:35:12.748323+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:50.279615+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:50.279615+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:37:04.686428+0000","scrub_duration":0.001448866,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748365+0000","last_change":"2024-09-14T04:33:59.335857+0000","last_active":"2024-09-14T04:35:13.748365+0000","last_peered":"2024-09-14T04:35:13.748365+0000","last_clean":"2024-09-14T04:35:13.748365+0000","last_became_active":"2024-09-14T04:33:59.334539+0000","last_became_peered":"2024-09-14T04:33:59.334539+0000","last_unstale":"2024-09-14T04:35:13.748365+0000","last_undegraded":"2024-09-14T04:35:13.748365+0000","last_fullsized":"2024-09-14T04:35:13.748365+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:53.748882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":160,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.749937+0000","last_change":"2024-09-14T04:27:20.239669+0000","last_active":"2024-09-14T04:35:13.749937+0000","last_peered":"2024-09-14T04:35:13.749937+0000","last_clean":"2024-09-14T04:35:13.749937+0000","last_became_active":"2024-09-14T04:27:20.239328+0000","last_became_peered":"2024-09-14T04:27:20.239328+0000","last_unstale":"2024-09-14T04:35:13.749937+0000","last_undegraded":"2024-09-14T04:35:13.749937+0000","last_fullsized":"2024-09-14T04:35:13.749937+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:32.696840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043261200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":800,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752942+0000","last_change":"2024-09-14T04:27:20.298034+0000","last_active":"2024-09-14T04:35:13.752942+0000","last_peered":"2024-09-14T04:35:13.752942+0000","last_clean":"2024-09-14T04:35:13.752942+0000","last_became_active":"2024-09-14T04:27:20.297633+0000","last_became_peered":"2024-09-14T04:27:20.297633+0000","last_unstale":"2024-09-14T04:35:13.752942+0000","last_undegraded":"2024-09-14T04:35:13.752942+0000","last_fullsized":"2024-09-14T04:35:13.752942+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:14:27.649199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"86'149","reported_seq":969,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756095+0000","last_change":"2024-09-14T04:33:59.340118+0000","last_active":"2024-09-14T04:35:13.756095+0000","last_peered":"2024-09-14T04:35:13.756095+0000","last_clean":"2024-09-14T04:35:13.756095+0000","last_became_active":"2024-09-14T04:33:59.339690+0000","last_became_peered":"2024-09-14T04:33:59.339690+0000","last_unstale":"2024-09-14T04:35:13.756095+0000","last_undegraded":"2024-09-14T04:35:13.756095+0000","last_fullsized":"2024-09-14T04:35:13.756095+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:52.546611+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:33:52.546611+0000","last_clean_scrub_stamp":"2024-09-14T04:33:52.546611+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:53.371259+0000","scrub_duration":0.001481095,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":64,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757781+0000","last_change":"2024-09-14T04:34:20.508620+0000","last_active":"2024-09-14T04:35:13.757781+0000","last_peered":"2024-09-14T04:35:13.757781+0000","last_clean":"2024-09-14T04:35:13.757781+0000","last_became_active":"2024-09-14T04:34:20.508315+0000","last_became_peered":"2024-09-14T04:34:20.508315+0000","last_unstale":"2024-09-14T04:35:13.757781+0000","last_undegraded":"2024-09-14T04:35:13.757781+0000","last_fullsized":"2024-09-14T04:35:13.757781+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:05:19.179517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748291+0000","last_change":"2024-09-14T04:33:59.345511+0000","last_active":"2024-09-14T04:35:13.748291+0000","last_peered":"2024-09-14T04:35:13.748291+0000","last_clean":"2024-09-14T04:35:13.748291+0000","last_became_active":"2024-09-14T04:33:59.345081+0000","last_became_peered":"2024-09-14T04:33:59.345081+0000","last_unstale":"2024-09-14T04:35:13.748291+0000","last_undegraded":"2024-09-14T04:35:13.748291+0000","last_fullsized":"2024-09-14T04:35:13.748291+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:14:14.641270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":393,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756852+0000","last_change":"2024-09-14T04:34:20.508544+0000","last_active":"2024-09-14T04:35:13.756852+0000","last_peered":"2024-09-14T04:35:13.756852+0000","last_clean":"2024-09-14T04:35:13.756852+0000","last_became_active":"2024-09-14T04:34:20.508251+0000","last_became_peered":"2024-09-14T04:34:20.508251+0000","last_unstale":"2024-09-14T04:35:13.756852+0000","last_undegraded":"2024-09-14T04:35:13.756852+0000","last_fullsized":"2024-09-14T04:35:13.756852+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:38.540818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"86'149","reported_seq":1318,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757487+0000","last_change":"2024-09-14T04:27:23.250388+0000","last_active":"2024-09-14T04:35:13.757487+0000","last_peered":"2024-09-14T04:35:13.757487+0000","last_clean":"2024-09-14T04:35:13.757487+0000","last_became_active":"2024-09-14T04:27:23.250051+0000","last_became_peered":"2024-09-14T04:27:23.250051+0000","last_unstale":"2024-09-14T04:35:13.757487+0000","last_undegraded":"2024-09-14T04:35:13.757487+0000","last_fullsized":"2024-09-14T04:35:13.757487+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":3,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:20.778793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:20.778793+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:15.915497+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":813,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752946+0000","last_change":"2024-09-14T04:33:59.346006+0000","last_active":"2024-09-14T04:35:13.752946+0000","last_peered":"2024-09-14T04:35:13.752946+0000","last_clean":"2024-09-14T04:35:13.752946+0000","last_became_active":"2024-09-14T04:33:59.336636+0000","last_became_peered":"2024-09-14T04:33:59.336636+0000","last_unstale":"2024-09-14T04:35:13.752946+0000","last_undegraded":"2024-09-14T04:35:13.752946+0000","last_fullsized":"2024-09-14T04:35:13.752946+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:42:49.996255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747914+0000","last_change":"2024-09-14T04:33:56.343398+0000","last_active":"2024-09-14T04:35:12.747914+0000","last_peered":"2024-09-14T04:35:12.747914+0000","last_clean":"2024-09-14T04:35:12.747914+0000","last_became_active":"2024-09-14T04:33:56.343047+0000","last_became_peered":"2024-09-14T04:33:56.343047+0000","last_unstale":"2024-09-14T04:35:12.747914+0000","last_undegraded":"2024-09-14T04:35:12.747914+0000","last_fullsized":"2024-09-14T04:35:12.747914+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:17.704720+0000","last_clean_scrub_stamp":"2024-09-14T03:41:17.704720+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:53:50.690046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032361299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":585,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744375+0000","last_change":"2024-09-14T04:28:19.256241+0000","last_active":"2024-09-14T04:35:12.744375+0000","last_peered":"2024-09-14T04:35:12.744375+0000","last_clean":"2024-09-14T04:35:12.744375+0000","last_became_active":"2024-09-14T04:28:19.255737+0000","last_became_peered":"2024-09-14T04:28:19.255737+0000","last_unstale":"2024-09-14T04:35:12.744375+0000","last_undegraded":"2024-09-14T04:35:12.744375+0000","last_fullsized":"2024-09-14T04:35:12.744375+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:16.598345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"86'149","reported_seq":862,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748336+0000","last_change":"2024-09-14T04:34:04.164795+0000","last_active":"2024-09-14T04:35:13.748336+0000","last_peered":"2024-09-14T04:35:13.748336+0000","last_clean":"2024-09-14T04:35:13.748336+0000","last_became_active":"2024-09-14T04:34:02.356941+0000","last_became_peered":"2024-09-14T04:34:02.356941+0000","last_unstale":"2024-09-14T04:35:13.748336+0000","last_undegraded":"2024-09-14T04:35:13.748336+0000","last_fullsized":"2024-09-14T04:35:13.748336+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:04.164744+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:04.164744+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:31:14.475505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":648,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.746599+0000","last_change":"2024-09-14T04:33:59.341470+0000","last_active":"2024-09-14T04:35:12.746599+0000","last_peered":"2024-09-14T04:35:12.746599+0000","last_clean":"2024-09-14T04:35:12.746599+0000","last_became_active":"2024-09-14T04:33:59.341164+0000","last_became_peered":"2024-09-14T04:33:59.341164+0000","last_unstale":"2024-09-14T04:35:12.746599+0000","last_undegraded":"2024-09-14T04:35:12.746599+0000","last_fullsized":"2024-09-14T04:35:12.746599+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:19:07.220907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'23","reported_seq":649,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.747316+0000","last_change":"2024-09-14T04:28:19.264292+0000","last_active":"2024-09-14T04:35:12.747316+0000","last_peered":"2024-09-14T04:35:12.747316+0000","last_clean":"2024-09-14T04:35:12.747316+0000","last_became_active":"2024-09-14T04:28:19.256967+0000","last_became_peered":"2024-09-14T04:28:19.256967+0000","last_unstale":"2024-09-14T04:35:12.747316+0000","last_undegraded":"2024-09-14T04:35:12.747316+0000","last_fullsized":"2024-09-14T04:35:12.747316+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:40.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"86'149","reported_seq":956,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:14.042036+0000","last_change":"2024-09-14T04:35:14.042036+0000","last_active":"2024-09-14T04:35:14.042036+0000","last_peered":"2024-09-14T04:35:14.042036+0000","last_clean":"2024-09-14T04:35:14.042036+0000","last_became_active":"2024-09-14T04:35:13.763481+0000","last_became_peered":"2024-09-14T04:35:13.763481+0000","last_unstale":"2024-09-14T04:35:14.042036+0000","last_undegraded":"2024-09-14T04:35:14.042036+0000","last_fullsized":"2024-09-14T04:35:14.042036+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:14.041928+0000","last_deep_scrub":"86'149","last_deep_scrub_stamp":"2024-09-14T04:08:36.230433+0000","last_clean_scrub_stamp":"2024-09-14T04:35:14.041928+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:51.392429+0000","scrub_duration":0.0016617579999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747852+0000","last_change":"2024-09-14T04:33:59.340411+0000","last_active":"2024-09-14T04:35:13.747852+0000","last_peered":"2024-09-14T04:35:13.747852+0000","last_clean":"2024-09-14T04:35:13.747852+0000","last_became_active":"2024-09-14T04:33:59.340172+0000","last_became_peered":"2024-09-14T04:33:59.340172+0000","last_unstale":"2024-09-14T04:35:13.747852+0000","last_undegraded":"2024-09-14T04:35:13.747852+0000","last_fullsized":"2024-09-14T04:35:13.747852+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:23:06.457371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":165,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737386+0000","last_change":"2024-09-14T04:27:20.251236+0000","last_active":"2024-09-14T04:35:12.737386+0000","last_peered":"2024-09-14T04:35:12.737386+0000","last_clean":"2024-09-14T04:35:12.737386+0000","last_became_active":"2024-09-14T04:27:20.236836+0000","last_became_peered":"2024-09-14T04:27:20.236836+0000","last_unstale":"2024-09-14T04:35:12.737386+0000","last_undegraded":"2024-09-14T04:35:12.737386+0000","last_fullsized":"2024-09-14T04:35:12.737386+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:45:14.509227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"86'149","reported_seq":913,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753770+0000","last_change":"2024-09-14T04:33:59.331662+0000","last_active":"2024-09-14T04:35:13.753770+0000","last_peered":"2024-09-14T04:35:13.753770+0000","last_clean":"2024-09-14T04:35:13.753770+0000","last_became_active":"2024-09-14T04:33:59.331251+0000","last_became_peered":"2024-09-14T04:33:59.331251+0000","last_unstale":"2024-09-14T04:35:13.753770+0000","last_undegraded":"2024-09-14T04:35:13.753770+0000","last_fullsized":"2024-09-14T04:35:13.753770+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:19:19.813989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:19:19.813989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:00:43.956655+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":439,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.758013+0000","last_change":"2024-09-14T04:35:13.756946+0000","last_active":"2024-09-14T04:35:13.758013+0000","last_peered":"2024-09-14T04:35:13.758013+0000","last_clean":"2024-09-14T04:35:13.758013+0000","last_became_active":"2024-09-14T04:35:13.756615+0000","last_became_peered":"2024-09-14T04:35:13.756615+0000","last_unstale":"2024-09-14T04:35:13.758013+0000","last_undegraded":"2024-09-14T04:35:13.758013+0000","last_fullsized":"2024-09-14T04:35:13.758013+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:25.517599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757291+0000","last_change":"2024-09-14T04:34:20.506624+0000","last_active":"2024-09-14T04:35:13.757291+0000","last_peered":"2024-09-14T04:35:13.757291+0000","last_clean":"2024-09-14T04:35:13.757291+0000","last_became_active":"2024-09-14T04:34:20.506213+0000","last_became_peered":"2024-09-14T04:34:20.506213+0000","last_unstale":"2024-09-14T04:35:13.757291+0000","last_undegraded":"2024-09-14T04:35:13.757291+0000","last_fullsized":"2024-09-14T04:35:13.757291+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:16.454233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"86'149","reported_seq":977,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737550+0000","last_change":"2024-09-14T04:34:03.233993+0000","last_active":"2024-09-14T04:35:12.737550+0000","last_peered":"2024-09-14T04:35:12.737550+0000","last_clean":"2024-09-14T04:35:12.737550+0000","last_became_active":"2024-09-14T04:34:02.355842+0000","last_became_peered":"2024-09-14T04:34:02.355842+0000","last_unstale":"2024-09-14T04:35:12.737550+0000","last_undegraded":"2024-09-14T04:35:12.737550+0000","last_fullsized":"2024-09-14T04:35:12.737550+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:03.233909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:03.233909+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:11:38.397295+0000","scrub_duration":0.0021438249999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":786,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753209+0000","last_change":"2024-09-14T04:24:07.982548+0000","last_active":"2024-09-14T04:35:13.753209+0000","last_peered":"2024-09-14T04:35:13.753209+0000","last_clean":"2024-09-14T04:35:13.753209+0000","last_became_active":"2024-09-14T04:24:07.981357+0000","last_became_peered":"2024-09-14T04:24:07.981357+0000","last_unstale":"2024-09-14T04:35:13.753209+0000","last_undegraded":"2024-09-14T04:35:13.753209+0000","last_fullsized":"2024-09-14T04:35:13.753209+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:58.448096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":912,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757073+0000","last_change":"2024-09-14T04:33:59.339494+0000","last_active":"2024-09-14T04:35:13.757073+0000","last_peered":"2024-09-14T04:35:13.757073+0000","last_clean":"2024-09-14T04:35:13.757073+0000","last_became_active":"2024-09-14T04:33:59.339188+0000","last_became_peered":"2024-09-14T04:33:59.339188+0000","last_unstale":"2024-09-14T04:35:13.757073+0000","last_undegraded":"2024-09-14T04:35:13.757073+0000","last_fullsized":"2024-09-14T04:35:13.757073+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:38:20.215115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"86'149","reported_seq":748,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744545+0000","last_change":"2024-09-14T04:34:08.551968+0000","last_active":"2024-09-14T04:35:12.744545+0000","last_peered":"2024-09-14T04:35:12.744545+0000","last_clean":"2024-09-14T04:35:12.744545+0000","last_became_active":"2024-09-14T04:34:02.358643+0000","last_became_peered":"2024-09-14T04:34:02.358643+0000","last_unstale":"2024-09-14T04:35:12.744545+0000","last_undegraded":"2024-09-14T04:35:12.744545+0000","last_fullsized":"2024-09-14T04:35:12.744545+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:08.551862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:08.551862+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:34.530388+0000","scrub_duration":0.001558928,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748098+0000","last_change":"2024-09-14T04:33:59.339591+0000","last_active":"2024-09-14T04:35:12.748098+0000","last_peered":"2024-09-14T04:35:12.748098+0000","last_clean":"2024-09-14T04:35:12.748098+0000","last_became_active":"2024-09-14T04:33:59.339122+0000","last_became_peered":"2024-09-14T04:33:59.339122+0000","last_unstale":"2024-09-14T04:35:12.748098+0000","last_undegraded":"2024-09-14T04:35:12.748098+0000","last_fullsized":"2024-09-14T04:35:12.748098+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:52.809466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":351,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757216+0000","last_change":"2024-09-14T04:34:20.656480+0000","last_active":"2024-09-14T04:35:13.757216+0000","last_peered":"2024-09-14T04:35:13.757216+0000","last_clean":"2024-09-14T04:35:13.757216+0000","last_became_active":"2024-09-14T04:34:20.656118+0000","last_became_peered":"2024-09-14T04:34:20.656118+0000","last_unstale":"2024-09-14T04:35:13.757216+0000","last_undegraded":"2024-09-14T04:35:13.757216+0000","last_fullsized":"2024-09-14T04:35:13.757216+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:29.581369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"86'149","reported_seq":546,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:25.176595+0000","last_change":"2024-09-14T04:35:25.176728+0000","last_active":"2024-09-14T04:35:25.176595+0000","last_peered":"2024-09-14T04:35:25.176595+0000","last_clean":"2024-09-14T04:35:25.176595+0000","last_became_active":"2024-09-14T04:35:13.760243+0000","last_became_peered":"2024-09-14T04:35:13.760243+0000","last_unstale":"2024-09-14T04:35:25.176595+0000","last_undegraded":"2024-09-14T04:35:25.176595+0000","last_fullsized":"2024-09-14T04:35:25.176595+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:35:25.176581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:35:25.176581+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:46.706478+0000","scrub_duration":0.002187488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":810,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752858+0000","last_change":"2024-09-14T04:27:20.251778+0000","last_active":"2024-09-14T04:35:13.752858+0000","last_peered":"2024-09-14T04:35:13.752858+0000","last_clean":"2024-09-14T04:35:13.752858+0000","last_became_active":"2024-09-14T04:27:20.248391+0000","last_became_peered":"2024-09-14T04:27:20.248391+0000","last_unstale":"2024-09-14T04:35:13.752858+0000","last_undegraded":"2024-09-14T04:35:13.752858+0000","last_fullsized":"2024-09-14T04:35:13.752858+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:32.105228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.747675+0000","last_change":"2024-09-14T04:33:59.343158+0000","last_active":"2024-09-14T04:35:13.747675+0000","last_peered":"2024-09-14T04:35:13.747675+0000","last_clean":"2024-09-14T04:35:13.747675+0000","last_became_active":"2024-09-14T04:33:59.342282+0000","last_became_peered":"2024-09-14T04:33:59.342282+0000","last_unstale":"2024-09-14T04:35:13.747675+0000","last_undegraded":"2024-09-14T04:35:13.747675+0000","last_fullsized":"2024-09-14T04:35:13.747675+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:50:05.754340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"587'150","reported_seq":751,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:29.251778+0000","last_change":"2024-09-14T04:35:29.251897+0000","last_active":"2024-09-14T04:35:29.251778+0000","last_peered":"2024-09-14T04:35:29.251778+0000","last_clean":"2024-09-14T04:35:29.251778+0000","last_became_active":"2024-09-14T04:35:13.761178+0000","last_became_peered":"2024-09-14T04:35:13.761178+0000","last_unstale":"2024-09-14T04:35:29.251778+0000","last_undegraded":"2024-09-14T04:35:29.251778+0000","last_fullsized":"2024-09-14T04:35:29.251778+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":5,"last_scrub":"587'150","last_scrub_stamp":"2024-09-14T04:35:29.251757+0000","last_deep_scrub":"587'150","last_deep_scrub_stamp":"2024-09-14T04:35:29.251757+0000","last_clean_scrub_stamp":"2024-09-14T04:35:29.251757+0000","objects_scrubbed":1,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:30.874703+0000","scrub_duration":0.0035824609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":319,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757523+0000","last_change":"2024-09-14T04:34:54.389583+0000","last_active":"2024-09-14T04:35:13.757523+0000","last_peered":"2024-09-14T04:35:13.757523+0000","last_clean":"2024-09-14T04:35:13.757523+0000","last_became_active":"2024-09-14T04:34:54.389299+0000","last_became_peered":"2024-09-14T04:34:54.389299+0000","last_unstale":"2024-09-14T04:35:13.757523+0000","last_undegraded":"2024-09-14T04:35:13.757523+0000","last_fullsized":"2024-09-14T04:35:13.757523+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:03.599428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":216,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754812+0000","last_change":"2024-09-14T04:33:59.346158+0000","last_active":"2024-09-14T04:35:13.754812+0000","last_peered":"2024-09-14T04:35:13.754812+0000","last_clean":"2024-09-14T04:35:13.754812+0000","last_became_active":"2024-09-14T04:33:59.345759+0000","last_became_peered":"2024-09-14T04:33:59.345759+0000","last_unstale":"2024-09-14T04:35:13.754812+0000","last_undegraded":"2024-09-14T04:35:13.754812+0000","last_fullsized":"2024-09-14T04:35:13.754812+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:51:25.669328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"86'149","reported_seq":559,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.756948+0000","last_change":"2024-09-14T04:34:21.324557+0000","last_active":"2024-09-14T04:35:13.756948+0000","last_peered":"2024-09-14T04:35:13.756948+0000","last_clean":"2024-09-14T04:35:13.756948+0000","last_became_active":"2024-09-14T04:34:20.509879+0000","last_became_peered":"2024-09-14T04:34:20.509879+0000","last_unstale":"2024-09-14T04:35:13.756948+0000","last_undegraded":"2024-09-14T04:35:13.756948+0000","last_fullsized":"2024-09-14T04:35:13.756948+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:21.324506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:21.324506+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:33.643045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":76,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754007+0000","last_change":"2024-09-14T04:34:53.789706+0000","last_active":"2024-09-14T04:35:13.754007+0000","last_peered":"2024-09-14T04:35:13.754007+0000","last_clean":"2024-09-14T04:35:13.754007+0000","last_became_active":"2024-09-14T04:34:53.789424+0000","last_became_peered":"2024-09-14T04:34:53.789424+0000","last_unstale":"2024-09-14T04:35:13.754007+0000","last_undegraded":"2024-09-14T04:35:13.754007+0000","last_fullsized":"2024-09-14T04:35:13.754007+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:49.777958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":814,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.752787+0000","last_change":"2024-09-14T04:29:38.008376+0000","last_active":"2024-09-14T04:35:13.752787+0000","last_peered":"2024-09-14T04:35:13.752787+0000","last_clean":"2024-09-14T04:35:13.752787+0000","last_became_active":"2024-09-14T04:29:38.004468+0000","last_became_peered":"2024-09-14T04:29:38.004468+0000","last_unstale":"2024-09-14T04:35:13.752787+0000","last_undegraded":"2024-09-14T04:35:13.752787+0000","last_fullsized":"2024-09-14T04:35:13.752787+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:41.908470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"86'149","reported_seq":481,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757745+0000","last_change":"2024-09-14T04:33:58.052578+0000","last_active":"2024-09-14T04:35:13.757745+0000","last_peered":"2024-09-14T04:35:13.757745+0000","last_clean":"2024-09-14T04:35:13.757745+0000","last_became_active":"2024-09-14T04:33:57.336734+0000","last_became_peered":"2024-09-14T04:33:57.336734+0000","last_unstale":"2024-09-14T04:35:13.757745+0000","last_undegraded":"2024-09-14T04:35:13.757745+0000","last_fullsized":"2024-09-14T04:35:13.757745+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:58.052531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:58.052531+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:11.321631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.748699+0000","last_change":"2024-09-14T04:33:59.336010+0000","last_active":"2024-09-14T04:35:12.748699+0000","last_peered":"2024-09-14T04:35:12.748699+0000","last_clean":"2024-09-14T04:35:12.748699+0000","last_became_active":"2024-09-14T04:33:59.335702+0000","last_became_peered":"2024-09-14T04:33:59.335702+0000","last_unstale":"2024-09-14T04:35:12.748699+0000","last_undegraded":"2024-09-14T04:35:12.748699+0000","last_fullsized":"2024-09-14T04:35:12.748699+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:57.635601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":452,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.761300+0000","last_change":"2024-09-14T04:35:13.760793+0000","last_active":"2024-09-14T04:35:13.761300+0000","last_peered":"2024-09-14T04:35:13.761300+0000","last_clean":"2024-09-14T04:35:13.761300+0000","last_became_active":"2024-09-14T04:35:13.760455+0000","last_became_peered":"2024-09-14T04:35:13.760455+0000","last_unstale":"2024-09-14T04:35:13.761300+0000","last_undegraded":"2024-09-14T04:35:13.761300+0000","last_fullsized":"2024-09-14T04:35:13.761300+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:19:11.363794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":200,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.744881+0000","last_change":"2024-09-14T04:25:07.963482+0000","last_active":"2024-09-14T04:35:12.744881+0000","last_peered":"2024-09-14T04:35:12.744881+0000","last_clean":"2024-09-14T04:35:12.744881+0000","last_became_active":"2024-09-14T04:25:07.963029+0000","last_became_peered":"2024-09-14T04:25:07.963029+0000","last_unstale":"2024-09-14T04:35:12.744881+0000","last_undegraded":"2024-09-14T04:35:12.744881+0000","last_fullsized":"2024-09-14T04:35:12.744881+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:25:59.074650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":461,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.765788+0000","last_change":"2024-09-14T04:35:13.764757+0000","last_active":"2024-09-14T04:35:13.765788+0000","last_peered":"2024-09-14T04:35:13.765788+0000","last_clean":"2024-09-14T04:35:13.765788+0000","last_became_active":"2024-09-14T04:35:13.763960+0000","last_became_peered":"2024-09-14T04:35:13.763960+0000","last_unstale":"2024-09-14T04:35:13.765788+0000","last_undegraded":"2024-09-14T04:35:13.765788+0000","last_fullsized":"2024-09-14T04:35:13.765788+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":587,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:39.810230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"86'149","reported_seq":967,"reported_epoch":586,"state":"active+clean","last_fresh":"2024-09-14T04:35:12.737252+0000","last_change":"2024-09-14T04:34:02.356446+0000","last_active":"2024-09-14T04:35:12.737252+0000","last_peered":"2024-09-14T04:35:12.737252+0000","last_clean":"2024-09-14T04:35:12.737252+0000","last_became_active":"2024-09-14T04:34:02.356063+0000","last_became_peered":"2024-09-14T04:34:02.356063+0000","last_unstale":"2024-09-14T04:35:12.737252+0000","last_undegraded":"2024-09-14T04:35:12.737252+0000","last_fullsized":"2024-09-14T04:35:12.737252+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:33:58.219433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:33:58.219433+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:19.836104+0000","scrub_duration":0.0013900010000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":389,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757177+0000","last_change":"2024-09-14T04:34:20.510111+0000","last_active":"2024-09-14T04:35:13.757177+0000","last_peered":"2024-09-14T04:35:13.757177+0000","last_clean":"2024-09-14T04:35:13.757177+0000","last_became_active":"2024-09-14T04:34:20.509761+0000","last_became_peered":"2024-09-14T04:34:20.509761+0000","last_unstale":"2024-09-14T04:35:13.757177+0000","last_undegraded":"2024-09-14T04:35:13.757177+0000","last_fullsized":"2024-09-14T04:35:13.757177+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:36.158698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":60,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.757604+0000","last_change":"2024-09-14T04:34:20.509644+0000","last_active":"2024-09-14T04:35:13.757604+0000","last_peered":"2024-09-14T04:35:13.757604+0000","last_clean":"2024-09-14T04:35:13.757604+0000","last_became_active":"2024-09-14T04:34:20.509138+0000","last_became_peered":"2024-09-14T04:34:20.509138+0000","last_unstale":"2024-09-14T04:35:13.757604+0000","last_undegraded":"2024-09-14T04:35:13.757604+0000","last_fullsized":"2024-09-14T04:35:13.757604+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:53:17.819229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"86'149","reported_seq":831,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.748152+0000","last_change":"2024-09-14T04:34:05.167904+0000","last_active":"2024-09-14T04:35:13.748152+0000","last_peered":"2024-09-14T04:35:13.748152+0000","last_clean":"2024-09-14T04:35:13.748152+0000","last_became_active":"2024-09-14T04:34:02.356949+0000","last_became_peered":"2024-09-14T04:34:02.356949+0000","last_unstale":"2024-09-14T04:35:13.748152+0000","last_undegraded":"2024-09-14T04:35:13.748152+0000","last_fullsized":"2024-09-14T04:35:13.748152+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"86'149","last_scrub_stamp":"2024-09-14T04:34:05.167845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:53.062469+0000","last_clean_scrub_stamp":"2024-09-14T04:34:05.167845+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:14:19.773382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":600,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.754660+0000","last_change":"2024-09-14T04:33:59.337174+0000","last_active":"2024-09-14T04:35:13.754660+0000","last_peered":"2024-09-14T04:35:13.754660+0000","last_clean":"2024-09-14T04:35:13.754660+0000","last_became_active":"2024-09-14T04:33:59.336862+0000","last_became_peered":"2024-09-14T04:33:59.336862+0000","last_unstale":"2024-09-14T04:35:13.754660+0000","last_undegraded":"2024-09-14T04:35:13.754660+0000","last_fullsized":"2024-09-14T04:35:13.754660+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:34.625113+0000","last_clean_scrub_stamp":"2024-09-14T03:42:34.625113+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:00:23.183522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":861,"reported_epoch":587,"state":"active+clean","last_fresh":"2024-09-14T04:35:13.753318+0000","last_change":"2024-09-14T04:33:56.348370+0000","last_active":"2024-09-14T04:35:13.753318+0000","last_peered":"2024-09-14T04:35:13.753318+0000","last_clean":"2024-09-14T04:35:13.753318+0000","last_became_active":"2024-09-14T04:33:56.338455+0000","last_became_peered":"2024-09-14T04:33:56.338455+0000","last_unstale":"2024-09-14T04:35:13.753318+0000","last_undegraded":"2024-09-14T04:35:13.753318+0000","last_fullsized":"2024-09-14T04:35:13.753318+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:42:33.619286+0000","last_clean_scrub_stamp":"2024-09-14T03:42:33.619286+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:55:12.047606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":23122,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27347,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4620,"ondisk_log_size":4620,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201947,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498560,"kb_used_data":8632,"kb_used_omap":1,"kb_used_meta":1489918,"kb_avail":92246592,"statfs":{"total":95995035648,"available":94460510208,"internally_reserved":0,"allocated":8839168,"data_stored":5081895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1525676491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463585,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1217332,"kb_used_data":8676,"kb_used_omap":1,"kb_used_meta":1208638,"kb_avail":92527820,"statfs":{"total":95995035648,"available":94748487680,"internally_reserved":0,"allocated":8884224,"data_stored":5146206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1237645767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":372,"seq":1597727834257,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":635692,"kb_used_data":8612,"kb_used_omap":5,"kb_used_meta":627066,"kb_avail":93109460,"statfs":{"total":95995035648,"available":95344087040,"internally_reserved":0,"allocated":8818688,"data_stored":5077313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6083,"internal_metadata":642115645},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986857,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722876,"kb_used_data":8364,"kb_used_omap":1,"kb_used_meta":1714494,"kb_avail":92022276,"statfs":{"total":95995035648,"available":94230810624,"internally_reserved":0,"allocated":8564736,"data_stored":4818864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1755642310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748885,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266284,"kb_used_data":8668,"kb_used_omap":4,"kb_used_meta":1257595,"kb_avail":92478868,"statfs":{"total":95995035648,"available":94698360832,"internally_reserved":0,"allocated":8876032,"data_stored":5142017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4564,"internal_metadata":1287777836},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":297,"seq":1275605287106,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650028,"kb_used_data":8612,"kb_used_omap":10,"kb_used_meta":641397,"kb_avail":93095124,"statfs":{"total":95995035648,"available":95329406976,"internally_reserved":0,"allocated":8818688,"data_stored":5077551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10665,"internal_metadata":656791127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":108,"seq":463856468429,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191012,"kb_used_data":8344,"kb_used_omap":5,"kb_used_meta":1182650,"kb_avail":92554140,"statfs":{"total":95995035648,"available":94775439360,"internally_reserved":0,"allocated":8544256,"data_stored":4812846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5793,"internal_metadata":1211033951},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":173,"seq":743029342544,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":996524,"kb_used_data":8672,"kb_used_omap":10,"kb_used_meta":987829,"kb_avail":92748628,"statfs":{"total":95995035648,"available":94974595072,"internally_reserved":0,"allocated":8880128,"data_stored":5142531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11054,"internal_metadata":1011537106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9460,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4199,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6227,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2972,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4489,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:36:06.893 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:36:06.894 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi001 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T04:36:07.078 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=0ba3c1977ceb43679e667d5be6f4877a Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi001 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T04:36:07.081 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-14T04:36:07.092 INFO:tasks.cephadm:Teardown begin 2024-09-14T04:36:07.092 DEBUG:teuthology.orchestra.run.smithi001:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:36:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:06 smithi012 ceph-mon[101180]: pgmap v1863: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:07.098 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:06 smithi012 ceph-mon[101180]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:07.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:06 smithi001 ceph-mon[107399]: pgmap v1863: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:07.126 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:06 smithi001 ceph-mon[107399]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:07.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:06 smithi001 ceph-mon[112165]: pgmap v1863: 103 pgs: 103 active+clean; 582 KiB data, 8.8 GiB used, 706 GiB / 715 GiB avail 2024-09-14T04:36:07.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:06 smithi001 ceph-mon[112165]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:36:07.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:36:07.185 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-14T04:36:07.186 DEBUG:teuthology.orchestra.run.smithi001:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T04:36:07.204 DEBUG:teuthology.orchestra.run.smithi012:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T04:36:07.231 INFO:tasks.cephadm:Stopping all daemons... 2024-09-14T04:36:07.231 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-14T04:36:07.231 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a 2024-09-14T04:36:07.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:07 smithi001 systemd[1]: Stopping Ceph mon.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:07.849 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:36:07.583+0000 7fc0054c5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:07.850 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a[107377]: 2024-09-14T04:36:07.583+0000 7fc0054c5640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-14T04:36:07.850 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE Bus STOPPING 2024-09-14T04:36:07.850 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:36:07.850 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE Bus STOPPED 2024-09-14T04:36:07.850 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE Bus STARTING 2024-09-14T04:36:07.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE Serving on http://:::9283 2024-09-14T04:36:07.851 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:07 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:07] ENGINE Bus STARTED 2024-09-14T04:36:08.145 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:07 smithi001 podman[379413]: 2024-09-14 04:36:07.849389373 +0000 UTC m=+0.403428139 container died 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T04:36:08.470 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:08 smithi001 podman[379413]: 2024-09-14 04:36:08.300578763 +0000 UTC m=+0.854617527 container cleanup 36cdffa077fb68cfb7613320e0d6e27aac30f6162efaa9ef93812391f2cdde8d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:36:08.471 INFO:journalctl@ceph.mon.a.smithi001.stdout:Sep 14 04:36:08 smithi001 bash[379413]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T04:36:08.804 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.a.service' 2024-09-14T04:36:08.843 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:08.844 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-14T04:36:08.844 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-14T04:36:08.844 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.c 2024-09-14T04:36:09.191 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:08 smithi001 systemd[1]: Stopping Ceph mon.c for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c[112142]: 2024-09-14T04:36:09.190+0000 7fe08e5b4640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c[112142]: 2024-09-14T04:36:09.190+0000 7fe08e5b4640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-14T04:36:09.471 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:09 smithi001 podman[379555]: 2024-09-14 04:36:09.405659595 +0000 UTC m=+0.332078014 container died a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T04:36:09.471 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE Bus STOPPING 2024-09-14T04:36:09.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:36:09.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE Bus STOPPED 2024-09-14T04:36:09.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE Bus STARTING 2024-09-14T04:36:09.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE Serving on http://:::9283 2024-09-14T04:36:09.472 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:09 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: [14/Sep/2024:04:36:09] ENGINE Bus STARTED 2024-09-14T04:36:10.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:09 smithi001 podman[379555]: 2024-09-14 04:36:09.906120016 +0000 UTC m=+0.832538467 container cleanup a4ab7249719a2d51e29a0f1f76b5be3a6465380c78bc4f3d98b676ffc1aa99d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T04:36:10.127 INFO:journalctl@ceph.mon.c.smithi001.stdout:Sep 14 04:36:09 smithi001 bash[379555]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-c 2024-09-14T04:36:10.343 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.c.service' 2024-09-14T04:36:10.379 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:10.379 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-14T04:36:10.379 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-14T04:36:10.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.b 2024-09-14T04:36:10.705 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:10 smithi012 systemd[1]: Stopping Ceph mon.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:11.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b[101156]: 2024-09-14T04:36:10.702+0000 7f2d7c5dd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:11.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b[101156]: 2024-09-14T04:36:10.702+0000 7f2d7c5dd640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-14T04:36:11.096 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:10 smithi012 podman[262476]: 2024-09-14 04:36:10.93937446 +0000 UTC m=+0.349982294 container died f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T04:36:11.595 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:11 smithi012 podman[262476]: 2024-09-14 04:36:11.411466115 +0000 UTC m=+0.822073935 container cleanup f246dcb03561dd2921488fc2c9fa8bd4c230e43535072b7f2145563d0aa49f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T04:36:11.596 INFO:journalctl@ceph.mon.b.smithi012.stdout:Sep 14 04:36:11 smithi012 bash[262476]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mon-b 2024-09-14T04:36:11.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mon.b.service' 2024-09-14T04:36:11.913 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:11.913 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-14T04:36:11.914 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-14T04:36:11.914 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y 2024-09-14T04:36:12.220 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:11 smithi001 systemd[1]: Stopping Ceph mgr.y for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:12.498 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:36:12.239+0000 7fca121be640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:12.498 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:12 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y[90266]: 2024-09-14T04:36:12.239+0000 7fca121be640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-14T04:36:12.499 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:12 smithi001 podman[379700]: 2024-09-14 04:36:12.271770307 +0000 UTC m=+0.155459489 container died 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-14T04:36:13.032 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:12 smithi001 podman[379700]: 2024-09-14 04:36:12.772048749 +0000 UTC m=+0.655737917 container cleanup 5403daa749017e8d10db6edeb7c8d954972e3ddb4965bd7ebc0002eeca554725 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T04:36:13.033 INFO:journalctl@ceph.mgr.y.smithi001.stdout:Sep 14 04:36:12 smithi001 bash[379700]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-y 2024-09-14T04:36:13.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-14T04:36:13.208 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:13.208 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-14T04:36:13.208 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-14T04:36:13.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x 2024-09-14T04:36:13.567 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 04:36:13 smithi012 systemd[1]: Stopping Ceph mgr.x for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:13.846 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 04:36:13 smithi012 podman[262606]: 2024-09-14 04:36:13.566339435 +0000 UTC m=+0.159236018 container died 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-a278231) 2024-09-14T04:36:14.335 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 04:36:13 smithi012 podman[262606]: 2024-09-14 04:36:13.957798135 +0000 UTC m=+0.550694719 container cleanup 23caad2752d84e13102ce3e71b7a6920c528bfd9abe9ccb13085d5e0e8ea05a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T04:36:14.336 INFO:journalctl@ceph.mgr.x.smithi012.stdout:Sep 14 04:36:13 smithi012 bash[262606]: ceph-5626d550-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T04:36:14.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-14T04:36:14.577 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:14.578 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-14T04:36:14.578 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-14T04:36:14.578 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0 2024-09-14T04:36:14.880 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:14 smithi001 systemd[1]: Stopping Ceph osd.0 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:15.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:36:14.878+0000 7f1f4255e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:15.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:36:14.878+0000 7f1f4255e640 -1 osd.0 587 *** Got signal Terminated *** 2024-09-14T04:36:15.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:14 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0[183472]: 2024-09-14T04:36:14.878+0000 7f1f4255e640 -1 osd.0 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:20.795 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:20 smithi001 podman[379843]: 2024-09-14 04:36:20.495566481 +0000 UTC m=+5.721077523 container died f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-14T04:36:21.220 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:20 smithi001 podman[379843]: 2024-09-14 04:36:20.952473396 +0000 UTC m=+6.177984436 container cleanup f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-14T04:36:21.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:20 smithi001 bash[379843]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0 2024-09-14T04:36:21.580 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379865]: 2024-09-14 04:36:21.220747343 +0000 UTC m=+0.726699669 container remove f56fa74c3238b028ba45c593ad2b4db0863127bec732f60cdc313aaf7c25084d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-14T04:36:21.933 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379956]: 2024-09-14 04:36:21.482909272 +0000 UTC m=+0.016375824 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:36:21.933 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379956]: 2024-09-14 04:36:21.580092939 +0000 UTC m=+0.113559481 container create bea28dd7f1de00320defb41e6bd00ed1e35fb4b30fe1ba530f16e1073ed576ab (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=main-a278231, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T04:36:21.933 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379956]: 2024-09-14 04:36:21.807681693 +0000 UTC m=+0.341148243 container init bea28dd7f1de00320defb41e6bd00ed1e35fb4b30fe1ba530f16e1073ed576ab (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-14T04:36:21.933 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379956]: 2024-09-14 04:36:21.812806364 +0000 UTC m=+0.346272906 container start bea28dd7f1de00320defb41e6bd00ed1e35fb4b30fe1ba530f16e1073ed576ab (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:36:21.934 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:21 smithi001 podman[379956]: 2024-09-14 04:36:21.932555548 +0000 UTC m=+0.466022094 container attach bea28dd7f1de00320defb41e6bd00ed1e35fb4b30fe1ba530f16e1073ed576ab (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:36:22.221 INFO:journalctl@ceph.osd.0.smithi001.stdout:Sep 14 04:36:22 smithi001 podman[379956]: 2024-09-14 04:36:22.03476242 +0000 UTC m=+0.568228962 container died bea28dd7f1de00320defb41e6bd00ed1e35fb4b30fe1ba530f16e1073ed576ab (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-14T04:36:22.715 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.0.service' 2024-09-14T04:36:22.757 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:22.757 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-14T04:36:22.757 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-14T04:36:22.757 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.1 2024-09-14T04:36:22.970 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:22 smithi001 systemd[1]: Stopping Ceph osd.1 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:23.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:23 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T04:36:23.065+0000 7f8f95af3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:23.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:23 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T04:36:23.065+0000 7f8f95af3640 -1 osd.1 587 *** Got signal Terminated *** 2024-09-14T04:36:23.471 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:23 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1[129379]: 2024-09-14T04:36:23.065+0000 7f8f95af3640 -1 osd.1 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:28.841 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:28 smithi001 podman[380084]: 2024-09-14 04:36:28.536486323 +0000 UTC m=+5.552180900 container died dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-14T04:36:29.220 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380084]: 2024-09-14 04:36:29.056651998 +0000 UTC m=+6.072346573 container cleanup dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:36:29.221 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 bash[380084]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1 2024-09-14T04:36:29.720 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380105]: 2024-09-14 04:36:29.308493664 +0000 UTC m=+0.768403012 container remove dc72545877526a77dd4e9df89aec1087a4de3bcbdec35731b5cb165060611103 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-14T04:36:30.037 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380196]: 2024-09-14 04:36:29.625378685 +0000 UTC m=+0.016727334 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:36:30.037 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380196]: 2024-09-14 04:36:29.725939406 +0000 UTC m=+0.117288060 container create efb23570d8ede1e683bca8b21e0f332c9db6a17036cee274730de839a2eb30de (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T04:36:30.037 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380196]: 2024-09-14 04:36:29.953697937 +0000 UTC m=+0.345046588 container init efb23570d8ede1e683bca8b21e0f332c9db6a17036cee274730de839a2eb30de (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T04:36:30.037 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:29 smithi001 podman[380196]: 2024-09-14 04:36:29.959222419 +0000 UTC m=+0.350571074 container start efb23570d8ede1e683bca8b21e0f332c9db6a17036cee274730de839a2eb30de (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T04:36:30.038 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:30 smithi001 podman[380196]: 2024-09-14 04:36:30.036811373 +0000 UTC m=+0.428160020 container attach efb23570d8ede1e683bca8b21e0f332c9db6a17036cee274730de839a2eb30de (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T04:36:30.449 INFO:journalctl@ceph.osd.1.smithi001.stdout:Sep 14 04:36:30 smithi001 podman[380196]: 2024-09-14 04:36:30.156805872 +0000 UTC m=+0.548154524 container died efb23570d8ede1e683bca8b21e0f332c9db6a17036cee274730de839a2eb30de (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:36:30.741 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.1.service' 2024-09-14T04:36:30.782 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:30.782 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-14T04:36:30.782 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-14T04:36:30.783 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2 2024-09-14T04:36:31.114 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:30 smithi001 systemd[1]: Stopping Ceph osd.2 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:31.470 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:36:31.111+0000 7f7b40fed640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:31.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:36:31.111+0000 7f7b40fed640 -1 osd.2 587 *** Got signal Terminated *** 2024-09-14T04:36:31.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:31 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2[264247]: 2024-09-14T04:36:31.111+0000 7f7b40fed640 -1 osd.2 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:36.954 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:36 smithi001 podman[380323]: 2024-09-14 04:36:36.634506899 +0000 UTC m=+5.626898464 container died a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-14T04:36:37.220 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 podman[380323]: 2024-09-14 04:36:37.110031465 +0000 UTC m=+6.102423072 container cleanup a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:36:37.221 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 bash[380323]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2 2024-09-14T04:36:37.720 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 podman[380345]: 2024-09-14 04:36:37.386592402 +0000 UTC m=+0.751253989 container remove a36d6f49f78527ad6a707763a5b5b121df0e334e5ada7516ae3492ce5a949fe5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T04:36:37.998 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 podman[380441]: 2024-09-14 04:36:37.637380021 +0000 UTC m=+0.016740820 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:36:37.999 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 podman[380441]: 2024-09-14 04:36:37.770856752 +0000 UTC m=+0.150217567 container create 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-a278231) 2024-09-14T04:36:37.999 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:37 smithi001 podman[380441]: 2024-09-14 04:36:37.998221061 +0000 UTC m=+0.377581850 container init 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T04:36:38.470 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:38 smithi001 podman[380441]: 2024-09-14 04:36:38.003621794 +0000 UTC m=+0.382982585 container start 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True) 2024-09-14T04:36:38.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:38 smithi001 podman[380441]: 2024-09-14 04:36:38.081298919 +0000 UTC m=+0.460659707 container attach 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-14T04:36:38.471 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:38 smithi001 podman[380441]: 2024-09-14 04:36:38.211002293 +0000 UTC m=+0.590363076 container died 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-09-14T04:36:38.819 INFO:journalctl@ceph.osd.2.smithi001.stdout:Sep 14 04:36:38 smithi001 podman[380485]: 2024-09-14 04:36:38.801915043 +0000 UTC m=+0.580972828 container remove 5952516c65e69dd91b5234f17cf00b83b91d6761af1eb99332a264bedd02d18a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True) 2024-09-14T04:36:38.822 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.2.service' 2024-09-14T04:36:38.862 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:38.862 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-14T04:36:38.863 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-14T04:36:38.863 DEBUG:teuthology.orchestra.run.smithi001:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.3 2024-09-14T04:36:39.183 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:38 smithi001 systemd[1]: Stopping Ceph osd.3 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:39.470 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:36:39.181+0000 7fefeda86640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:39.470 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:36:39.181+0000 7fefeda86640 -1 osd.3 587 *** Got signal Terminated *** 2024-09-14T04:36:39.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:39 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:36:39.181+0000 7fefeda86640 -1 osd.3 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:41.846 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:41 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:41.568+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:42.970 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:42 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:36:42.631+0000 7fefea09b640 -1 osd.3 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.346766+0000 front 2024-09-14T04:36:18.346741+0000 (oldest deadline 2024-09-14T04:36:41.846719+0000) 2024-09-14T04:36:43.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:42 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:42.617+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:43.640 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:43.381+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:43.970 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:43 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3[141689]: 2024-09-14T04:36:43.647+0000 7fefea09b640 -1 osd.3 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.346766+0000 front 2024-09-14T04:36:18.346741+0000 (oldest deadline 2024-09-14T04:36:41.846719+0000) 2024-09-14T04:36:44.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:43 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:43.639+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:44.351 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:44.119+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:44.655 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:44.350+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:45.056 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:44 smithi001 podman[380569]: 2024-09-14 04:36:44.785597082 +0000 UTC m=+5.697030358 container died e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:36:45.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:44 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:44.654+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:45.470 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:45 smithi001 podman[380569]: 2024-09-14 04:36:45.262920615 +0000 UTC m=+6.174353882 container cleanup e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:36:45.471 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:45 smithi001 bash[380569]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3 2024-09-14T04:36:45.595 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:45.342+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:45.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:45.141+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:45.874 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:45 smithi001 podman[380600]: 2024-09-14 04:36:45.53981654 +0000 UTC m=+0.751246535 container remove e1e9c9c85c38f87c850a82e6504b034a04e336f79c73bdb5182e7f73d63cd1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T04:36:46.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:45 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:45.675+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:46.143 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:45 smithi001 podman[380695]: 2024-09-14 04:36:45.873946656 +0000 UTC m=+0.105071627 container create de91665940b68a47ff74ce1f5b228635217660098e6e294fe5c9c3227065825d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T04:36:46.143 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:45 smithi001 podman[380695]: 2024-09-14 04:36:45.785438289 +0000 UTC m=+0.016563224 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:36:46.143 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:46 smithi001 podman[380695]: 2024-09-14 04:36:46.068021387 +0000 UTC m=+0.299146323 container init de91665940b68a47ff74ce1f5b228635217660098e6e294fe5c9c3227065825d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T04:36:46.143 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:46 smithi001 podman[380695]: 2024-09-14 04:36:46.073226439 +0000 UTC m=+0.304351365 container start de91665940b68a47ff74ce1f5b228635217660098e6e294fe5c9c3227065825d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:36:46.143 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:46 smithi001 podman[380695]: 2024-09-14 04:36:46.142675843 +0000 UTC m=+0.373800788 container attach de91665940b68a47ff74ce1f5b228635217660098e6e294fe5c9c3227065825d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:36:46.470 INFO:journalctl@ceph.osd.3.smithi001.stdout:Sep 14 04:36:46 smithi001 podman[380695]: 2024-09-14 04:36:46.266431724 +0000 UTC m=+0.497556656 container died de91665940b68a47ff74ce1f5b228635217660098e6e294fe5c9c3227065825d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T04:36:46.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:46.330+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:46.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:46.123+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:46.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:46.152+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:47.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:46 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:46.696+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:47.170 DEBUG:teuthology.orchestra.run.smithi001:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.3.service' 2024-09-14T04:36:47.218 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:47.218 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-14T04:36:47.219 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-14T04:36:47.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.4 2024-09-14T04:36:47.553 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:47.304+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:47.553 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:47.154+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:47.553 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:47.124+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:47.554 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:47 smithi012 systemd[1]: Stopping Ceph osd.4 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:47.830 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:47.733+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:47.830 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:47.551+0000 7f20104d9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:47.831 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:47.551+0000 7f20104d9640 -1 osd.4 587 *** Got signal Terminated *** 2024-09-14T04:36:47.831 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:47.551+0000 7f20104d9640 -1 osd.4 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:48.096 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.829Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.096 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.830Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.096 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.832Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.834Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.835Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.838Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.097 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:36:47 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:36:47.838Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.1:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.1:8765: connect: connection refused" 2024-09-14T04:36:48.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:48 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:48.271+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:48.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:48 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:48.152+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:48.597 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:48 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:48.181+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:49.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:48 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:48.726+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:49.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:49 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:49.118+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:49.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:49 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:49.260+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:49.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:49 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:49.223+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:49.597 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:49 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:49.223+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:50.070 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:49 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:49.751+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:50.345 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:50.069+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:50.346 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:50.262+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:50.346 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:50.214+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:50.346 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:50.214+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:51.064 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:50.746+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:51.065 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:50 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:50.746+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:51.346 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:51.299+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:51.346 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:51.063+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:51.346 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:51.229+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:51.347 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:51.229+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:52.044 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:51.696+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:52.045 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:51 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:51.696+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:52.318 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:52.043+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.399018+0000 front 2024-09-14T04:36:19.399129+0000 (oldest deadline 2024-09-14T04:36:45.299096+0000) 2024-09-14T04:36:52.318 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4[38978]: 2024-09-14T04:36:52.043+0000 7f2008ae5640 -1 osd.4 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:25.800355+0000 front 2024-09-14T04:36:25.800449+0000 (oldest deadline 2024-09-14T04:36:51.700322+0000) 2024-09-14T04:36:52.319 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:52.250+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:52.319 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:52.250+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:52.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:52.317+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:52.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:52.317+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:53.019 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:52.738+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:53.019 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:52 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:52.738+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:53.279 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:53 smithi012 podman[262753]: 2024-09-14 04:36:53.018645138 +0000 UTC m=+5.588639961 container died 8cf884c205fc5b6166114cedb69b5f2280918a69c523a91d9cd44035a911234f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T04:36:53.279 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:53.258+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:53.279 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:53.258+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:53.595 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:53 smithi012 podman[262753]: 2024-09-14 04:36:53.499142963 +0000 UTC m=+6.069137787 container cleanup 8cf884c205fc5b6166114cedb69b5f2280918a69c523a91d9cd44035a911234f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:36:53.596 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:53 smithi012 bash[262753]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4 2024-09-14T04:36:53.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:53.277+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:53.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:53.277+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:54.096 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:53 smithi012 podman[262774]: 2024-09-14 04:36:53.759183569 +0000 UTC m=+0.731590202 container remove 8cf884c205fc5b6166114cedb69b5f2280918a69c523a91d9cd44035a911234f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-14T04:36:54.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:53.715+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:54.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:53 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:53.715+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:54.396 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:54.306+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:54.397 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:54.306+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:54.397 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:54.247+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:54.397 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:54.247+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:54.397 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.030884573 +0000 UTC m=+0.018584433 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:36:54.398 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.151860404 +0000 UTC m=+0.139560268 container create 30702a554c60e0b9230ecddf9f94dc0d5624dfbdc9fa2fd5cd9ec0ddf75a7917 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-14T04:36:54.738 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.396160809 +0000 UTC m=+0.383860669 container init 30702a554c60e0b9230ecddf9f94dc0d5624dfbdc9fa2fd5cd9ec0ddf75a7917 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-14T04:36:54.739 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.401471474 +0000 UTC m=+0.389171334 container start 30702a554c60e0b9230ecddf9f94dc0d5624dfbdc9fa2fd5cd9ec0ddf75a7917 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T04:36:54.739 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.529471473 +0000 UTC m=+0.517171334 container attach 30702a554c60e0b9230ecddf9f94dc0d5624dfbdc9fa2fd5cd9ec0ddf75a7917 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T04:36:55.033 INFO:journalctl@ceph.osd.4.smithi012.stdout:Sep 14 04:36:54 smithi012 podman[262865]: 2024-09-14 04:36:54.738248492 +0000 UTC m=+0.725948353 container died 30702a554c60e0b9230ecddf9f94dc0d5624dfbdc9fa2fd5cd9ec0ddf75a7917 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-14T04:36:55.033 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:54.745+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:55.033 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:54 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:54.745+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:55.291 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:55.242+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:55.292 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:55.242+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:55.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.4.service' 2024-09-14T04:36:55.366 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:36:55.367 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-14T04:36:55.367 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-14T04:36:55.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5 2024-09-14T04:36:55.595 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:55.320+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:55.320+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:55.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 systemd[1]: Stopping Ceph osd.5 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:36:56.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:55.746+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:56.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:55.746+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:56.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:55.688+0000 7f79acbef640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:36:56.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:55.688+0000 7f79acbef640 -1 osd.5 587 *** Got signal Terminated *** 2024-09-14T04:36:56.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:55 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:55.688+0000 7f79acbef640 -1 osd.5 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:36:56.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:56.233+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:56.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:56.233+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:56.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:56.359+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:56.597 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:56.359+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:57.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:56.787+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:57.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:56 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:56.787+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:57.595 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:57.370+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:57.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:57.370+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:57.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:57.213+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:57.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:57.213+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:58.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:57.789+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:58.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:57 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:57.789+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:58.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:58.405+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:58.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:58.405+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:58.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:58.223+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:58.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:58.223+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:58.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:58.223+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:36:59.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:58.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:36:59.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:58.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:36:59.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:58 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:58.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:36:59.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:59.377+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:36:59.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:36:59.377+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:36:59.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:59.261+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:36:59.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:59.261+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:36:59.597 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:36:59.261+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:00.255 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:59.851+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:00.255 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:59.851+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:00.255 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:36:59 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:36:59.851+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:00.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:37:00.396+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:17.679550+0000 front 2024-09-14T04:36:17.679793+0000 (oldest deadline 2024-09-14T04:36:42.379468+0000) 2024-09-14T04:37:00.596 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5[210269]: 2024-09-14T04:37:00.396+0000 7f79a9203640 -1 osd.5 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:27.680525+0000 front 2024-09-14T04:36:27.680516+0000 (oldest deadline 2024-09-14T04:36:51.780385+0000) 2024-09-14T04:37:00.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:00.253+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:00.597 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:00.253+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:00.597 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:00.253+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:01.180 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:00.852+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:01.181 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:00.852+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:01.181 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:00 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:00.852+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:01.471 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:01 smithi012 podman[262992]: 2024-09-14 04:37:01.180138303 +0000 UTC m=+5.612391489 container died f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:37:01.471 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:01.251+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:01.472 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:01.251+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:01.472 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:01.251+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:01.813 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:01 smithi012 podman[262992]: 2024-09-14 04:37:01.585711919 +0000 UTC m=+6.017965095 container cleanup f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-14T04:37:01.813 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:01 smithi012 bash[262992]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5 2024-09-14T04:37:02.096 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:01 smithi012 podman[263012]: 2024-09-14 04:37:01.812034147 +0000 UTC m=+0.630042130 container remove f9282eee7359c82360ec86cfe3a7fced1a3b42b41d5058601d18d2afd5b95ef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:37:02.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:01.858+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:02.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:01.858+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:02.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:01 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:01.858+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:02.484 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.127474853 +0000 UTC m=+0.016490146 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:37:02.484 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.254726689 +0000 UTC m=+0.143742007 container create e51f561ae9440f91869fe0b981411173840735b1eb4c653096aaa396fd3748ca (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T04:37:02.485 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:02.257+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:02.485 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:02.257+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:02.485 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:02.257+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:02.820 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.482126246 +0000 UTC m=+0.371141528 container init e51f561ae9440f91869fe0b981411173840735b1eb4c653096aaa396fd3748ca (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-14T04:37:02.820 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.487664504 +0000 UTC m=+0.376679788 container start e51f561ae9440f91869fe0b981411173840735b1eb4c653096aaa396fd3748ca (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T04:37:02.821 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.565084366 +0000 UTC m=+0.454099653 container attach e51f561ae9440f91869fe0b981411173840735b1eb4c653096aaa396fd3748ca (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:37:02.821 INFO:journalctl@ceph.osd.5.smithi012.stdout:Sep 14 04:37:02 smithi012 podman[263107]: 2024-09-14 04:37:02.676693016 +0000 UTC m=+0.565708305 container died e51f561ae9440f91869fe0b981411173840735b1eb4c653096aaa396fd3748ca (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-14T04:37:03.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:02.818+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:03.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:02.818+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:03.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:02 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:02.818+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:03.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.5.service' 2024-09-14T04:37:03.349 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:37:03.350 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-14T04:37:03.350 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-14T04:37:03.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.6 2024-09-14T04:37:03.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.278+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:03.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.278+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:03.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.278+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:03.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 systemd[1]: Stopping Ceph osd.6 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:04.096 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.698+0000 7fd164c5b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:37:04.096 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.698+0000 7fd164c5b640 -1 osd.6 587 *** Got signal Terminated *** 2024-09-14T04:37:04.096 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:03.698+0000 7fd164c5b640 -1 osd.6 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:37:04.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:03.822+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:04.097 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:03.822+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:04.097 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:03 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:03.822+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:04.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:04.327+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:04.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:04.327+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:04.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:04.327+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:05.327 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:04.844+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:05.327 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:04.844+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:05.327 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:04 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:04.844+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:05.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:05.326+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:05.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:05.326+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:05.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:05.326+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:06.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:05.823+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:06.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:05.823+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:06.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:05 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:05.823+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:06.805 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:06.361+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:06.805 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:06.361+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:06.806 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:06.361+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:07.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:06.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:07.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:06.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:07.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:06 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:06.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:07.813 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:07.394+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:07.814 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:07.394+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:07.814 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:07.394+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:07.814 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:07.394+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:42.577299+0000 front 2024-09-14T04:36:42.577262+0000 (oldest deadline 2024-09-14T04:37:06.677161+0000) 2024-09-14T04:37:08.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:07.812+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:08.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:07.812+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:08.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:07 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:07.812+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:08.773 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:08.389+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:18.074034+0000 front 2024-09-14T04:36:18.073789+0000 (oldest deadline 2024-09-14T04:36:43.373906+0000) 2024-09-14T04:37:08.774 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:08.389+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:23.374355+0000 front 2024-09-14T04:36:23.374527+0000 (oldest deadline 2024-09-14T04:36:48.674376+0000) 2024-09-14T04:37:08.774 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:08.389+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:34.475919+0000 front 2024-09-14T04:36:34.475951+0000 (oldest deadline 2024-09-14T04:36:57.975794+0000) 2024-09-14T04:37:08.774 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6[51703]: 2024-09-14T04:37:08.389+0000 7fd15d267640 -1 osd.6 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:42.577299+0000 front 2024-09-14T04:36:42.577262+0000 (oldest deadline 2024-09-14T04:37:06.677161+0000) 2024-09-14T04:37:09.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:08.772+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:09.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:08.772+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:09.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:08 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:08.772+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:09.595 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:09 smithi012 podman[263234]: 2024-09-14 04:37:09.302032723 +0000 UTC m=+5.719054951 container died 31884087247e149f62056d72c584016ccdc5a6eb310c886b4e194a2826a13b1f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:37:09.898 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:09 smithi012 podman[263234]: 2024-09-14 04:37:09.704429916 +0000 UTC m=+6.121452143 container cleanup 31884087247e149f62056d72c584016ccdc5a6eb310c886b4e194a2826a13b1f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-14T04:37:09.898 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:09 smithi012 bash[263234]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6 2024-09-14T04:37:09.899 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:09 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:09.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:09.899 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:09 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:09.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:09.899 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:09 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:09.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:10.327 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:09 smithi012 podman[263259]: 2024-09-14 04:37:09.897709597 +0000 UTC m=+0.582711574 container remove 31884087247e149f62056d72c584016ccdc5a6eb310c886b4e194a2826a13b1f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-14T04:37:10.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.230399132 +0000 UTC m=+0.016650810 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:37:10.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.340079983 +0000 UTC m=+0.126331657 container create ebca7aaf66f950e6d821697fed6e53f4843a60e5187ce17035abbea751009ae3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=reef-e4e227b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T04:37:10.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.534139643 +0000 UTC m=+0.320391318 container init ebca7aaf66f950e6d821697fed6e53f4843a60e5187ce17035abbea751009ae3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-14T04:37:10.596 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.538993569 +0000 UTC m=+0.325245249 container start ebca7aaf66f950e6d821697fed6e53f4843a60e5187ce17035abbea751009ae3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T04:37:11.029 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.640191466 +0000 UTC m=+0.426443145 container attach ebca7aaf66f950e6d821697fed6e53f4843a60e5187ce17035abbea751009ae3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:37:11.030 INFO:journalctl@ceph.osd.6.smithi012.stdout:Sep 14 04:37:10 smithi012 podman[263351]: 2024-09-14 04:37:10.737751733 +0000 UTC m=+0.524003412 container died ebca7aaf66f950e6d821697fed6e53f4843a60e5187ce17035abbea751009ae3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:37:11.030 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:10.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:11.030 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:10.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:11.030 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:10.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:11.030 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:10 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:10.804+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:11.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.6.service' 2024-09-14T04:37:11.356 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:37:11.356 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-14T04:37:11.356 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-14T04:37:11.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.7 2024-09-14T04:37:11.595 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 systemd[1]: Stopping Ceph osd.7 for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:12.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.684+0000 7f77ae413640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:37:12.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.684+0000 7f77ae413640 -1 osd.7 587 *** Got signal Terminated *** 2024-09-14T04:37:12.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.684+0000 7f77ae413640 -1 osd.7 587 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:37:12.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.831+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:12.097 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.831+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:12.097 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.831+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:12.097 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:11 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:11.831+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:13.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:12.784+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:13.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:12.784+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:13.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:12.784+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:13.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:12 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:12.784+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:14.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:13.815+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:14.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:13.815+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:14.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:13.815+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:14.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:13 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:13.815+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:15.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:14.799+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:15.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:14.799+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:15.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:14.799+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:15.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:14 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:14.799+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:16.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:15.751+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6806 osd.0 since back 2024-09-14T04:36:19.719944+0000 front 2024-09-14T04:36:19.719929+0000 (oldest deadline 2024-09-14T04:36:40.819813+0000) 2024-09-14T04:37:16.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:15.751+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6814 osd.1 since back 2024-09-14T04:36:26.021539+0000 front 2024-09-14T04:36:26.021553+0000 (oldest deadline 2024-09-14T04:36:50.721407+0000) 2024-09-14T04:37:16.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:15.751+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6822 osd.2 since back 2024-09-14T04:36:36.022574+0000 front 2024-09-14T04:36:36.022503+0000 (oldest deadline 2024-09-14T04:36:58.322490+0000) 2024-09-14T04:37:16.096 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:15 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7[57817]: 2024-09-14T04:37:15.751+0000 7f77a6a1f640 -1 osd.7 587 heartbeat_check: no reply from 172.21.15.1:6830 osd.3 since back 2024-09-14T04:36:44.124197+0000 front 2024-09-14T04:36:44.124290+0000 (oldest deadline 2024-09-14T04:37:10.024223+0000) 2024-09-14T04:37:17.508 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:17 smithi012 podman[263479]: 2024-09-14 04:37:17.235306455 +0000 UTC m=+5.647482928 container died e3977061966099f3caa1b5bdf751b71916db651b7c1f6868a695d6a1f9e357d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T04:37:17.792 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:17 smithi012 podman[263479]: 2024-09-14 04:37:17.581739774 +0000 UTC m=+5.993916216 container cleanup e3977061966099f3caa1b5bdf751b71916db651b7c1f6868a695d6a1f9e357d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:37:17.792 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:17 smithi012 bash[263479]: ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7 2024-09-14T04:37:18.095 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:17 smithi012 podman[263512]: 2024-09-14 04:37:17.79111166 +0000 UTC m=+0.544238312 container remove e3977061966099f3caa1b5bdf751b71916db651b7c1f6868a695d6a1f9e357d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:37:18.503 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.097888669 +0000 UTC m=+0.016409950 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:37:18.503 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.208545678 +0000 UTC m=+0.127066955 container create f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T04:37:18.503 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.427592941 +0000 UTC m=+0.346114222 container init f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-14T04:37:18.503 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.432388096 +0000 UTC m=+0.350909375 container start f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T04:37:18.503 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.502276495 +0000 UTC m=+0.420797774 container attach f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-14T04:37:18.846 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:18 smithi012 podman[263604]: 2024-09-14 04:37:18.623537297 +0000 UTC m=+0.542058579 container died f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, ceph=True) 2024-09-14T04:37:19.175 INFO:journalctl@ceph.osd.7.smithi012.stdout:Sep 14 04:37:19 smithi012 podman[263648]: 2024-09-14 04:37:19.172557838 +0000 UTC m=+0.542654902 container remove f4819fc19171f475684fb9a3bc52dd4af9b0392124b865acd86d49e902a6455e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-14T04:37:19.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@osd.7.service' 2024-09-14T04:37:19.238 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:37:19.238 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-14T04:37:19.238 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-14T04:37:19.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a 2024-09-14T04:37:19.569 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 systemd[1]: Stopping Ceph prometheus.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:19.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T04:37:19.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T04:37:19.846 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.569Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T04:37:19.847 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.571Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T04:37:19.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.571Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T04:37:19.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a[88138]: ts=2024-09-14T04:37:19.571Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T04:37:19.848 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 podman[263733]: 2024-09-14 04:37:19.579946164 +0000 UTC m=+0.115140797 container died ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:20.202 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 podman[263733]: 2024-09-14 04:37:19.98804606 +0000 UTC m=+0.523240693 container cleanup ebbb50c30a92adfe5dcf399492d7b601ab02905847034966f0c28cac77a5c51e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:20.202 INFO:journalctl@ceph.prometheus.a.smithi012.stdout:Sep 14 04:37:19 smithi012 bash[263733]: ceph-5626d550-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T04:37:20.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5626d550-724a-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-14T04:37:20.377 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:37:20.377 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-14T04:37:20.377 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5626d550-724a-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T04:37:20.540 INFO:teuthology.orchestra.run.smithi001.stdout:Deleting cluster with fsid: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T04:37:22.219 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:21 smithi001 systemd[1]: Stopping Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:22.470 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:22 smithi001 ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a[102819]: ts=2024-09-14T04:37:22.218Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T04:37:22.471 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:22 smithi001 podman[381120]: 2024-09-14 04:37:22.290022861 +0000 UTC m=+0.166418443 container died 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:22.970 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:22 smithi001 podman[381120]: 2024-09-14 04:37:22.739811504 +0000 UTC m=+0.616207087 container cleanup 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:22.970 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:22 smithi001 bash[381120]: ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T04:37:23.331 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:23 smithi001 podman[381137]: 2024-09-14 04:37:23.0745149 +0000 UTC m=+0.784807198 container remove 4117f985a870be8c98cd7b04dc3f44f4c251897eefc927fa67eb57d3a298d49e (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:23.331 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:23 smithi001 podman[381137]: 2024-09-14 04:37:23.183684363 +0000 UTC m=+0.893976661 volume remove 9aea3f7410907508d54b6c643ee1845eba8b24a669828f12b16c68f8931dbaf1 2024-09-14T04:37:23.331 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:23 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T04:37:23.331 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:23 smithi001 systemd[1]: Stopped Ceph alertmanager.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:37:23.331 INFO:journalctl@ceph.alertmanager.a.smithi001.stdout:Sep 14 04:37:23 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 4.418s CPU time. 2024-09-14T04:37:23.720 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:23 smithi001 systemd[1]: Stopping Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:24.162 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:23 smithi001 podman[381251]: 2024-09-14 04:37:23.783161065 +0000 UTC m=+0.122294436 container died 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:24.162 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 podman[381251]: 2024-09-14 04:37:24.161913339 +0000 UTC m=+0.501046750 container cleanup 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:24.422 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 bash[381251]: ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T04:37:24.720 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 podman[381267]: 2024-09-14 04:37:24.421422373 +0000 UTC m=+0.641907560 container remove 6234cc31ec43c07cf0118ddbd4937fa0fa5ebc27737338c0a7fc044be2601111 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:37:24.720 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T04:37:24.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T04:37:24.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 systemd[1]: Stopped Ceph node-exporter.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:37:24.721 INFO:journalctl@ceph.node-exporter.a.smithi001.stdout:Sep 14 04:37:24 smithi001 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 10.102s CPU time. 2024-09-14T04:37:31.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5626d550-724a-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T04:37:31.947 INFO:teuthology.orchestra.run.smithi012.stdout:Deleting cluster with fsid: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T04:37:33.570 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 systemd[1]: Stopping Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:33.570 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=server t=2024-09-14T04:37:33.538800551Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-14T04:37:33.571 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=ticker t=2024-09-14T04:37:33.539205782Z level=info msg=stopped last_tick=2024-09-14T04:37:30Z 2024-09-14T04:37:33.571 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=tracing t=2024-09-14T04:37:33.539237543Z level=info msg="Closing tracing" 2024-09-14T04:37:33.571 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a[92774]: logger=grafana-apiserver t=2024-09-14T04:37:33.539275049Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-14T04:37:33.845 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:33 smithi012 podman[264153]: 2024-09-14 04:37:33.569975468 +0000 UTC m=+0.123206309 container died d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:37:34.205 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 podman[264153]: 2024-09-14 04:37:34.00308565 +0000 UTC m=+0.556316491 container cleanup d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:37:34.205 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 bash[264153]: ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a 2024-09-14T04:37:34.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 podman[264170]: 2024-09-14 04:37:34.204436494 +0000 UTC m=+0.629057382 container remove d361d041a0ad648e4cba47443fa04f86cacd90c90629b4faf3df6864d052d0c2 (image=quay.io/ceph/grafana:10.4.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:37:34.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-14T04:37:34.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 systemd[1]: Stopped Ceph grafana.a for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:37:34.540 INFO:journalctl@ceph.grafana.a.smithi012.stdout:Sep 14 04:37:34 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@grafana.a.service: Consumed 34.816s CPU time. 2024-09-14T04:37:34.845 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:34 smithi012 systemd[1]: Stopping Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968... 2024-09-14T04:37:34.846 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:34 smithi012 podman[264282]: 2024-09-14 04:37:34.787028907 +0000 UTC m=+0.103459716 container died a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:37:35.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 podman[264282]: 2024-09-14 04:37:35.123988607 +0000 UTC m=+0.440419457 container cleanup a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:37:35.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 bash[264282]: ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T04:37:35.346 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 podman[264298]: 2024-09-14 04:37:35.291730589 +0000 UTC m=+0.504202721 container remove a27184543b46ff797b89a1f340f6757018a898a0d9e967e65041c56f18c13573 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-5626d550-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:37:35.347 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T04:37:35.845 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T04:37:35.846 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 systemd[1]: Stopped Ceph node-exporter.b for 5626d550-724a-11ef-bceb-c7b262605968. 2024-09-14T04:37:35.846 INFO:journalctl@ceph.node-exporter.b.smithi012.stdout:Sep 14 04:37:35 smithi012 systemd[1]: ceph-5626d550-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 9.277s CPU time. 2024-09-14T04:37:39.628 DEBUG:teuthology.orchestra.run.smithi001:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:37:39.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:37:39.692 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-14T04:37:39.694 DEBUG:teuthology.misc:Transferring archived files from smithi001:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi001/crash 2024-09-14T04:37:39.695 DEBUG:teuthology.orchestra.run.smithi001:> sudo tar c -f - -C /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash -- . 2024-09-14T04:37:39.747 INFO:teuthology.orchestra.run.smithi001.stderr:tar: /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T04:37:39.747 INFO:teuthology.orchestra.run.smithi001.stderr:tar: Error is not recoverable: exiting now 2024-09-14T04:37:39.749 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi012/crash 2024-09-14T04:37:39.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash -- . 2024-09-14T04:37:39.784 INFO:teuthology.orchestra.run.smithi012.stderr:tar: /var/lib/ceph/5626d550-724a-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T04:37:39.784 INFO:teuthology.orchestra.run.smithi012.stderr:tar: Error is not recoverable: exiting now 2024-09-14T04:37:39.785 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-14T04:37:39.785 DEBUG:teuthology.orchestra.run.smithi001:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-14T04:37:39.867 INFO:teuthology.orchestra.run.smithi001.stdout:2024-09-14T03:41:19.662041+0000 mon.a (mon.0) 696 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:37:39.872 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-14T04:37:39.872 INFO:tasks.cephadm:Compressing logs... 2024-09-14T04:37:39.873 DEBUG:teuthology.orchestra.run.smithi001:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T04:37:39.875 DEBUG:teuthology.orchestra.run.smithi012:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T04:37:39.933 INFO:teuthology.orchestra.run.smithi012.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-14T04:37:39.934 INFO:teuthology.orchestra.run.smithi012.stderr:: No such file or directory 2024-09-14T04:37:39.934 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi012.nnhswt.log 2024-09-14T04:37:39.934 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.7.log: /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi012.nnhswt.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr: 83.7% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi012.nnhswt.log.gz 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-14T04:37:39.937 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log 2024-09-14T04:37:39.938 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-14T04:37:39.938 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-14T04:37:39.938 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T04:37:39.949 INFO:teuthology.orchestra.run.smithi001.stderr:find: ‘/var/log/rbd-target-api’gzip -5: No such file or directory --verbose 2024-09-14T04:37:39.949 INFO:teuthology.orchestra.run.smithi001.stderr: -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-14T04:37:39.949 INFO:teuthology.orchestra.run.smithi001.stderr:gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi001.ytjgbj.log 2024-09-14T04:37:39.953 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-14T04:37:39.953 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi001.ytjgbj.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr: 86.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mds.foo.smithi001.ytjgbj.log.gz 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-14T04:37:39.954 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-14T04:37:39.955 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log 2024-09-14T04:37:39.955 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-14T04:37:39.955 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T04:37:39.966 INFO:teuthology.orchestra.run.smithi001.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.8% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T04:37:40.109 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 90.4% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T04:37:40.313 INFO:teuthology.orchestra.run.smithi001.stderr: 94.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T04:37:40.397 INFO:teuthology.orchestra.run.smithi012.stderr: 85.9% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T04:37:40.404 INFO:teuthology.orchestra.run.smithi012.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T04:37:40.489 INFO:teuthology.orchestra.run.smithi012.stderr: 87.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T04:37:40.891 INFO:teuthology.orchestra.run.smithi012.stderr: 91.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T04:37:41.459 INFO:teuthology.orchestra.run.smithi001.stderr: 92.1% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T04:37:41.993 INFO:teuthology.orchestra.run.smithi012.stderr: 89.8% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-14T04:37:42.246 INFO:teuthology.orchestra.run.smithi001.stderr: 94.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T04:37:42.284 INFO:teuthology.orchestra.run.smithi001.stderr: 91.1% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T04:37:44.993 INFO:teuthology.orchestra.run.smithi001.stderr: 89.6% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-14T04:37:56.676 INFO:teuthology.orchestra.run.smithi012.stderr: 92.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-14T04:38:11.636 INFO:teuthology.orchestra.run.smithi001.stderr: 91.7% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-14T04:38:41.061 INFO:teuthology.orchestra.run.smithi001.stderr: 90.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-14T05:07:48.179 INFO:teuthology.orchestra.run.smithi012.stderr: 93.3% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-14T05:09:16.373 INFO:teuthology.orchestra.run.smithi012.stderr: 93.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-14T05:10:05.383 INFO:teuthology.orchestra.run.smithi012.stderr: 93.4% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-14T05:10:19.127 INFO:teuthology.orchestra.run.smithi012.stderr: 92.5% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-14T05:10:19.129 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-14T05:10:19.129 INFO:teuthology.orchestra.run.smithi012.stderr:real 32m39.238s 2024-09-14T05:10:19.129 INFO:teuthology.orchestra.run.smithi012.stderr:user 10m59.033s 2024-09-14T05:10:19.129 INFO:teuthology.orchestra.run.smithi012.stderr:sys 0m32.681s 2024-09-14T05:12:15.429 INFO:teuthology.orchestra.run.smithi001.stderr: 92.8% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-14T05:13:06.769 INFO:teuthology.orchestra.run.smithi001.stderr: 93.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-14T05:15:57.200 INFO:teuthology.orchestra.run.smithi001.stderr: 93.2% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-14T05:16:06.648 INFO:teuthology.orchestra.run.smithi001.stderr: 92.9% -- replaced with /var/log/ceph/5626d550-724a-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-14T05:16:06.651 INFO:teuthology.orchestra.run.smithi001.stderr: 2024-09-14T05:16:06.651 INFO:teuthology.orchestra.run.smithi001.stderr:real 38m26.761s 2024-09-14T05:16:06.651 INFO:teuthology.orchestra.run.smithi001.stderr:user 13m34.800s 2024-09-14T05:16:06.651 INFO:teuthology.orchestra.run.smithi001.stderr:sys 0m38.240s 2024-09-14T05:16:06.652 INFO:tasks.cephadm:Archiving logs... 2024-09-14T05:16:06.652 DEBUG:teuthology.misc:Transferring archived files from smithi001:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi001/log 2024-09-14T05:16:06.673 DEBUG:teuthology.orchestra.run.smithi001:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T05:18:02.004 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi012/log 2024-09-14T05:18:02.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T05:19:34.249 INFO:tasks.cephadm:Removing cluster... 2024-09-14T05:19:34.250 DEBUG:teuthology.orchestra.run.smithi001:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5626d550-724a-11ef-bceb-c7b262605968 --force 2024-09-14T05:19:34.599 INFO:teuthology.orchestra.run.smithi001.stdout:Deleting cluster with fsid: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T05:19:35.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5626d550-724a-11ef-bceb-c7b262605968 --force 2024-09-14T05:19:35.775 INFO:teuthology.orchestra.run.smithi012.stdout:Deleting cluster with fsid: 5626d550-724a-11ef-bceb-c7b262605968 2024-09-14T05:19:36.659 INFO:tasks.cephadm:Removing cephadm ... 2024-09-14T05:19:36.659 DEBUG:teuthology.orchestra.run.smithi001:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T05:19:36.679 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T05:19:36.694 INFO:tasks.cephadm:Teardown complete 2024-09-14T05:19:36.695 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T05:19:36.703 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T05:19:36.704 DEBUG:teuthology.orchestra.run.smithi001:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T05:19:36.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T05:19:36.830 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T05:19:36.839 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T05:19:36.840 DEBUG:teuthology.orchestra.run.smithi001:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T05:19:36.843 DEBUG:teuthology.orchestra.run.smithi012:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T05:19:36.859 INFO:teuthology.orchestra.run.smithi001.stderr:bash: line 1: ntpq: command not found 2024-09-14T05:19:36.859 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:=============================================================================== 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:^* hv01.front.sepia.ceph.com 3 6 377 57 -103us[ -90us] +/- 47ms 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:^+ hv02.front.sepia.ceph.com 3 8 377 63 +179us[ +193us] +/- 62ms 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 57 -87us[ -87us] +/- 65ms 2024-09-14T05:19:36.888 INFO:teuthology.orchestra.run.smithi001.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T05:19:36.894 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T05:19:36.894 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-09-14T05:19:36.895 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv01.front.sepia.ceph.com 3 6 377 59 -89us[ -80us] +/- 47ms 2024-09-14T05:19:36.895 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 60 +183us[ +192us] +/- 62ms 2024-09-14T05:19:36.895 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 50 -35us[ -35us] +/- 65ms 2024-09-14T05:19:36.895 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T05:19:36.896 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T05:19:36.906 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T05:19:36.906 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T05:19:36.915 DEBUG:teuthology.orchestra.run.smithi001:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T05:19:37.045 DEBUG:teuthology.orchestra.run.smithi012:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T05:19:37.155 DEBUG:teuthology.orchestra.run.smithi001:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T05:19:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:19:37.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T05:19:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:19:37.218 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T05:19:37.228 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T05:19:37.239 INFO:teuthology.task.internal:Duration was 6631.018317 seconds 2024-09-14T05:19:37.239 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T05:19:37.251 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T05:19:37.251 DEBUG:teuthology.orchestra.run.smithi001:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T05:19:37.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T05:19:37.309 INFO:teuthology.orchestra.run.smithi001.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T05:19:37.332 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T05:19:37.781 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T05:19:37.782 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi001.front.sepia.ceph.com 2024-09-14T05:19:37.782 DEBUG:teuthology.orchestra.run.smithi001:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T05:19:37.806 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi012.front.sepia.ceph.com 2024-09-14T05:19:37.807 DEBUG:teuthology.orchestra.run.smithi012:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T05:19:37.840 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T05:19:37.841 DEBUG:teuthology.orchestra.run.smithi001:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T05:19:37.851 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T05:19:37.908 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T05:19:37.909 DEBUG:teuthology.orchestra.run.smithi001:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T05:19:37.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T05:19:39.867 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T05:19:40.670 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T05:19:40.671 DEBUG:teuthology.orchestra.run.smithi001:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T05:19:40.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T05:19:40.748 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T05:19:40.846 DEBUG:teuthology.orchestra.run.smithi001:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:19:40.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:19:40.876 INFO:teuthology.orchestra.run.smithi001.stdout:kernel.core_pattern = core 2024-09-14T05:19:40.888 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = core 2024-09-14T05:19:40.913 DEBUG:teuthology.orchestra.run.smithi001:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:19:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:19:40.971 DEBUG:teuthology.orchestra.run.smithi012:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:19:40.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:19:40.989 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T05:19:41.001 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T05:19:41.002 DEBUG:teuthology.misc:Transferring archived files from smithi001:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi001 2024-09-14T05:19:41.002 DEBUG:teuthology.orchestra.run.smithi001:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T05:19:41.093 DEBUG:teuthology.misc:Transferring archived files from smithi012:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904532/remote/smithi012 2024-09-14T05:19:41.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T05:19:41.144 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T05:19:41.145 DEBUG:teuthology.orchestra.run.smithi001:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T05:19:41.148 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T05:19:41.194 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T05:19:41.206 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T05:19:41.207 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T05:19:41.218 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T05:19:41.218 DEBUG:teuthology.orchestra.run.smithi001:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T05:19:41.221 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T05:19:41.237 INFO:teuthology.orchestra.run.smithi001.stdout: 265324 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 05:19 /home/ubuntu/cephtest 2024-09-14T05:19:41.237 INFO:teuthology.orchestra.run.smithi001.stdout: 787323 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 04:33 /home/ubuntu/cephtest/mnt.0 2024-09-14T05:19:41.238 INFO:teuthology.orchestra.run.smithi001.stdout: 813295 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 04:33 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T05:19:41.238 INFO:teuthology.orchestra.run.smithi001.stdout: 6433339 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 04:33 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T05:19:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:19:41.239 INFO:teuthology.orchestra.run.smithi001.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-14T05:19:41.239 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi001 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T05:19:41.246 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T05:19:41.258 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T05:19:41.271 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi001 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T05:19:41.280 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 6631.018316745758 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi001 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=0ba3c1977ceb43679e667d5be6f4877a status: fail success: false 2024-09-14T05:19:41.281 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T05:19:41.291 INFO:teuthology.orchestra.run.smithi012.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 05:19 /home/ubuntu/cephtest 2024-09-14T05:19:41.376 INFO:teuthology.run:FAIL